Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/monitoring): prevent double webhook send and backfill deliver… (#3618)
tomsideguide3d1d2ca37daf
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsindex bd90cae70..dae0eea83 100644--- a/apps/api/src/controllers/v2/monitor.ts+++ b/apps/api/src/controllers/v2/monitor.ts@@ -35,6 +35,12 @@ import {trackMonitorConfiguredInterest,trackMonitorDeactivatedInterest,} from "../../services/monitoring/interest";+import {+ getLatestWebhookLog,+ getLatestWebhookLogsByJob,+ type WebhookLogRow,+} from "../../services/webhook/logs";+import { WebhookEvent } from "../../services/webhook";const logger = _logger.child({ module: "monitor-controller" });@@ -86,6 +92,38 @@ function serializeMonitor(monitor: any) {};}+function overlayWebhookLog<T extends { notificationStatus: any }>(+ serialized: T,+ log: WebhookLogRow | null,+): T {+ if (!log) return serialized;+ const notificationStatus =+ serialized.notificationStatus &&+ typeof serialized.notificationStatus === "object"+ ? serialized.notificationStatus+ : {};+ const existing =+ notificationStatus.webhook && typeof notificationStatus.webhook === "object"+ ? notificationStatus.webhook+ : {};+ return {+ ...serialized,+ notificationStatus: {+ ...notificationStatus,+ webhook: {+ ...existing,+ attempted: true,+ success: log.success === true,+ delivered: log.success === true,+ queued: false,+ statusCode: log.status_code ?? undefined,+ error: log.error ?? undefined,+ deliveredAt: log.created_at,+ },+ },+ };+}+function serializeCheck(check: any) {return {id: check.id,@@ -357,9 +395,19 @@ export async function listMonitorChecksController(status: query.status,});+ const webhookLogs = await getLatestWebhookLogsByJob({+ jobIds: checks.map(c => c.id),+ event: WebhookEvent.MONITOR_CHECK_COMPLETED,+ });+res.status(200).json({success: true,- data: checks.map(serializeCheck),+ data: checks.map(check =>+ overlayWebhookLog(+ serializeCheck(check),+ webhookLogs.get(check.id) ?? null,+ ),+ ),});}@@ -375,7 +423,7 @@ export async function getMonitorCheckController(return res.status(404).json({ success: false, error: "Check not found" });}- const [pages, totalPagesForFilter] = await Promise.all([+ const [pages, totalPagesForFilter, webhookLog] = await Promise.all([listMonitorCheckPages({teamId: req.auth.team_id,monitorId,@@ -388,6 +436,10 @@ export async function getMonitorCheckController(checkId,status: query.status,}),+ getLatestWebhookLog({+ jobId: checkId,+ event: WebhookEvent.MONITOR_CHECK_COMPLETED,+ }),]);const pagesWithDiffs = await Promise.all(@@ -442,7 +494,7 @@ export async function getMonitorCheckController(success: true,next,data: {- ...serializeCheck(check),+ ...overlayWebhookLog(serializeCheck(check), webhookLog),pages: pagesWithDiffs,next,},diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 6ad84a5e3..6122eb4c0 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -28,6 +28,7 @@ import {import { createWebhookSender, WebhookEvent } from "../webhook";import { sendMonitoringEmailSummary } from "../notification/monitoring_email";import {+ getMonitorCheck,getMonitorForUpdate,getMonitorPage,countMonitorCheckPages,@@ -64,6 +65,24 @@ export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };const MONITOR_DEFAULT_WAIT_FOR_MS = 5000;const MONITOR_MAX_WAIT_FOR_MS = 60000;const MONITOR_MAX_TIMEOUT_MS = MONITOR_MAX_WAIT_FOR_MS * 2;+const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;+const TERMINAL_CHECK_STATUSES = new Set([+ "completed",+ "partial",+ "failed",+ "skipped_overlap",+]);++async function claimMonitorNotification(checkId: string): Promise<boolean> {+ const result = await redisEvictConnection.set(+ `monitor-check-notify:${checkId}`,+ "1",+ "EX",+ MONITOR_NOTIFY_CLAIM_TTL_SECONDS,+ "NX",+ );+ return result === "OK";+}type PageResult = MonitorCheckPageInsert & {emailStatus?: string;@@ -837,12 +856,24 @@ export async function processMonitorCheckJob(throw new Error("Monitor not found");}+ const initialCheck = await getMonitorCheck(+ job.teamId,+ job.monitorId,+ job.checkId,+ );+ if (!initialCheck) {+ throw new Error("Monitor check not found");+ }+ if (TERMINAL_CHECK_STATUSES.has(initialCheck.status)) {+ return;+ }+await markMonitorRunning({monitorId: monitor.id,checkId: job.checkId,});- let check = await updateMonitorCheck(job.checkId, {+ let check: MonitorCheckRow = await updateMonitorCheck(job.checkId, {status: "running",started_at: new Date().toISOString(),});@@ -909,15 +940,37 @@ export async function processMonitorCheckJob(error: error instanceof Error ? error.message : String(error),});- await sendNotifications({- monitor,- check,- pages: [],- }).catch(err =>- logger.warn("Failed to send monitor failure notifications", {- error: err,- }),- );+ if ((await claimMonitorNotification(check.id).catch(error => {+ logger.warn("Failed to claim monitor notification; continuing without dedupe", {+ error,+ monitorId: monitor.id,+ checkId: check.id,+ });+ return true;+ }))) {+ const notificationStatus = await sendNotifications({+ monitor,+ check,+ pages: [],+ }).catch(err => {+ logger.warn("Failed to send monitor failure notifications", {+ error: err,+ });+ return null;+ });+ if (notificationStatus) {+ check = await updateMonitorCheck(check.id, {+ notification_status: notificationStatus,+ }).catch(updateError => {+ logger.warn("Failed to record monitor failure notification status", {+ error: updateError,+ monitorId: monitor.id,+ checkId: check.id,+ });+ return check;+ });+ }+ }await updateMonitorScheduleAfterRun({monitor,@@ -1066,46 +1119,49 @@ async function failStaleMonitorCheck(params: {error,});- const notificationStatus = await sendNotifications({- monitor: params.monitor,- check: finalized,- pages: [],- }).catch(notificationError => {- logger.warn("Failed to send stale monitor check notifications", {- error: notificationError,- monitorId: params.monitor.id,- checkId: params.check.id,+ let withNotifications = finalized;+ if (await claimMonitorNotification(params.check.id)) {+ const notificationStatus = await sendNotifications({+ monitor: params.monitor,+ check: finalized,+ pages: [],+ }).catch(notificationError => {+ logger.warn("Failed to send stale monitor check notifications", {+ error: notificationError,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ });+ return {+ webhook: {+ attempted: !!params.monitor.webhook,+ success: false,+ error:+ notificationError instanceof Error+ ? notificationError.message+ : String(notificationError),+ },+ email: {+ attempted: !!params.monitor.notification?.email?.enabled,+ success: false,+ error:+ notificationError instanceof Error+ ? notificationError.message+ : String(notificationError),+ },+ };});- return {- webhook: {- attempted: !!params.monitor.webhook,- success: false,- error:- notificationError instanceof Error- ? notificationError.message- : String(notificationError),- },- email: {- attempted: !!params.monitor.notification?.email?.enabled,- success: false,- error:- notificationError instanceof Error- ? notificationError.message- : String(notificationError),- },- };- });- const withNotifications = await updateMonitorCheck(params.check.id, {- notification_status: notificationStatus,- }).catch(updateError => {- logger.warn("Failed to record stale monitor check notification status", {- error: updateError,- monitorId: params.monitor.id,- checkId: params.check.id,+ withNotifications = await updateMonitorCheck(params.check.id, {+ notification_status: notificationStatus,+ }).catch(updateError => {+ logger.warn("Failed to record stale monitor check notification status", {+ error: updateError,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ });+ return finalized;});- return finalized;- });+ }if (params.monitor.current_check_id === params.check.id) {await updateMonitorScheduleAfterRun({@@ -1218,60 +1274,62 @@ export async function reconcileRunningMonitorChecks(});}- let notificationStatus: { webhook?: unknown; email?: unknown } | null =- null;- try {- const pages = (await listMonitorCheckPages({- teamId: monitor.team_id,- monitorId: monitor.id,- checkId: check.id,- limit: 100,- skip: 0,- })) as PageResult[];-- notificationStatus = await sendNotifications({- monitor,- check: finalized,- pages,- });+ if (await claimMonitorNotification(check.id)) {+ let notificationStatus: { webhook?: unknown; email?: unknown } | null =+ null;+ try {+ const pages = (await listMonitorCheckPages({+ teamId: monitor.team_id,+ monitorId: monitor.id,+ checkId: check.id,+ limit: 100,+ skip: 0,+ })) as PageResult[];++ notificationStatus = await sendNotifications({+ monitor,+ check: finalized,+ pages,+ });- finalized = await updateMonitorCheck(check.id, {- notification_status: notificationStatus,- webhook_payload: notificationStatus.webhook- ? { summary: toSummaryObject(finalized) }- : null,- email_payload: notificationStatus.email- ? { summary: toSummaryObject(finalized) }- : null,- });- } catch (error) {- logger.warn("Failed to send monitor check notifications", {- monitorId: monitor.id,- checkId: finalized.id,- error,- });- notificationStatus = {- webhook: {- attempted: !!monitor.webhook,- success: false,- error: error instanceof Error ? error.message : String(error),- },- email: {- attempted: !!monitor.notification?.email?.enabled,- success: false,- error: error instanceof Error ? error.message : String(error),- },- };- finalized = await updateMonitorCheck(check.id, {- notification_status: notificationStatus,- }).catch(updateError => {- logger.warn("Failed to record monitor check notification failure", {+ finalized = await updateMonitorCheck(check.id, {+ notification_status: notificationStatus,+ webhook_payload: notificationStatus.webhook+ ? { summary: toSummaryObject(finalized) }+ : null,+ email_payload: notificationStatus.email+ ? { summary: toSummaryObject(finalized) }+ : null,+ });+ } catch (error) {+ logger.warn("Failed to send monitor check notifications", {monitorId: monitor.id,checkId: finalized.id,- error: updateError,+ error,});- return finalized;- });+ notificationStatus = {+ webhook: {+ attempted: !!monitor.webhook,+ success: false,+ error: error instanceof Error ? error.message : String(error),+ },+ email: {+ attempted: !!monitor.notification?.email?.enabled,+ success: false,+ error: error instanceof Error ? error.message : String(error),+ },+ };+ finalized = await updateMonitorCheck(check.id, {+ notification_status: notificationStatus,+ }).catch(updateError => {+ logger.warn("Failed to record monitor check notification failure", {+ monitorId: monitor.id,+ checkId: finalized.id,+ error: updateError,+ });+ return finalized;+ });+ }}await updateMonitorScheduleAfterRun({diff --git a/apps/api/src/services/webhook/logs.ts b/apps/api/src/services/webhook/logs.tsnew file mode 100644index 000000000..af69ac1d1--- /dev/null+++ b/apps/api/src/services/webhook/logs.ts@@ -0,0 +1,68 @@+import { supabase_rr_service } from "../supabase";+import { logger as _logger } from "../../lib/logger";++export type WebhookLogRow = {+ id: string;+ success: boolean;+ error: string | null;+ status_code: number | null;+ latency_ms: number | null;+ url: string;+ event: string;+ created_at: string;+};++export async function getLatestWebhookLog(params: {+ jobId: string;+ event: string;+}): Promise<WebhookLogRow | null> {+ const { data, error } = await supabase_rr_service+ .from("webhook_logs")+ .select("id,success,error,status_code,latency_ms,url,event,created_at")+ .eq("crawl_id", params.jobId)+ .eq("event", params.event)+ .order("created_at", { ascending: false })+ .limit(1)+ .maybeSingle();+ if (error) {+ _logger.warn("Failed to fetch latest webhook log", {+ module: "webhook-logs",+ jobId: params.jobId,+ event: params.event,+ error,+ });+ return null;+ }+ return data as WebhookLogRow | null;+}++export async function getLatestWebhookLogsByJob(params: {+ jobIds: string[];+ event: string;+}): Promise<Map<string, WebhookLogRow>> {+ const result = new Map<string, WebhookLogRow>();+ if (params.jobIds.length === 0) return result;++ const { data, error } = await supabase_rr_service+ .from("webhook_logs")+ .select(+ "id,success,error,status_code,latency_ms,url,event,created_at,crawl_id",+ )+ .in("crawl_id", params.jobIds)+ .eq("event", params.event)+ .order("created_at", { ascending: false });++ if (error) {+ _logger.warn("Failed to fetch webhook logs batch", {+ module: "webhook-logs",+ event: params.event,+ error,+ });+ return result;+ }++ for (const row of (data ?? []) as (WebhookLogRow & { crawl_id: string })[]) {+ if (!result.has(row.crawl_id)) result.set(row.crawl_id, row);+ }+ return result;+}