Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(monitoring): various fixes

mogery3afe6df1f48f
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index 4b1cc32ba..217925c92 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -8,7 +8,7 @@ describe("monitoring runner", () => {
describe("isMonitorCheckStale", () => {
const now = new Date("2026-05-06T12:00:00.000Z");
- it("returns true when a running check is at least 24 hours old", () => {
+ it("returns true when a running check is at least 1 hour old", () => {
expect(
isMonitorCheckStale(
{
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index e5371dafe..42a74a479 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -52,15 +52,65 @@ import type {
import { withMarkdownFormat } from "./types";
import { redisEvictConnection } from "../redis";
import type { MonitorCheckJobData } from "./queue";
+import {
+ MONITOR_CHECK_STALE_ERROR,
+ isMonitorCheckStale,
+ MONITOR_CHECK_STALE_TIMEOUT_MS,
+} from "./stale";
const logger = _logger.child({ module: "monitoring-runner" });
const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-export const MONITOR_CHECK_STALE_TIMEOUT_MS = 24 * 60 * 60 * 1000;
+export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };
type PageResult = MonitorCheckPageInsert & {
emailStatus?: string;
};
+type MonitorTargetRun =
+ | {
+ targetId: string;
+ type: "scrape";
+ expectedJobs: string[];
+ }
+ | {
+ targetId: string;
+ type: "crawl";
+ crawlId: string;
+ };
+
+function createMonitorTargetRun(target: MonitorTarget): MonitorTargetRun {
+ if (target.type === "scrape") {
+ return {
+ targetId: target.id,
+ type: "scrape",
+ expectedJobs: target.urls.map(() => uuidv7()),
+ };
+ }
+
+ return {
+ targetId: target.id,
+ type: "crawl",
+ crawlId: uuidv7(),
+ };
+}
+
+function recoverScrapeTargetRunsFromMonitor(
+ monitor: MonitorRow,
+): MonitorTargetRun[] | null {
+ if (!monitor.targets.every(target => target.type === "scrape")) {
+ return null;
+ }
+
+ return monitor.targets.map(target => ({
+ targetId: target.id,
+ type: "scrape" as const,
+ expectedJobs:
+ target.type === "scrape"
+ ? target.urls.map((_, index) => `recovered:${target.id}:${index}`)
+ : [],
+ }));
+}
+
function withMonitorScrapeDefaults(
options: Record<string, unknown>,
): ScrapeOptions {
@@ -548,12 +598,28 @@ async function sendNotifications(params: {
webhook: params.monitor.webhook as any,
v0: false,
});
- await sender?.send(WebhookEvent.MONITOR_CHECK_COMPLETED, {
- success: params.check.status === "completed",
- data: payload,
- error: params.check.error ?? undefined,
- });
- webhookStatus = { attempted: true, success: true };
+ try {
+ const result = await sender?.send(WebhookEvent.MONITOR_CHECK_COMPLETED, {
+ success: params.check.status === "completed",
+ data: payload,
+ error: params.check.error ?? undefined,
+ awaitWebhook: true,
+ });
+ webhookStatus = {
+ attempted: result?.attempted ?? false,
+ success: result?.delivered === true,
+ delivered: result?.delivered === true,
+ queued: result?.queued === true,
+ skipped: result?.skipped === true,
+ };
+ } catch (error) {
+ webhookStatus = {
+ attempted: true,
+ success: false,
+ delivered: false,
+ error: error instanceof Error ? error.message : String(error),
+ };
+ }
}
const emailStatus = await sendMonitoringEmailSummary({
@@ -579,14 +645,14 @@ async function enqueueMonitorScrapeTarget(params: {
monitor: MonitorRow;
check: MonitorCheckRow;
target: MonitorTarget;
-}): Promise<{ targetId: string; type: "scrape"; expectedJobs: string[] }> {
+ targetRun: Extract<MonitorTargetRun, { type: "scrape" }>;
+}): Promise<Extract<MonitorTargetRun, { type: "scrape" }>> {
if (params.target.type !== "scrape") {
throw new Error("Expected scrape target");
}
- const expectedJobs: string[] = [];
- for (const url of params.target.urls) {
- const scrapeId = uuidv7();
+ for (const [index, url] of params.target.urls.entries()) {
+ const scrapeId = params.targetRun.expectedJobs[index];
const scrapeOptions = scrapeRequestSchema.parse({
url,
...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),
@@ -632,22 +698,22 @@ async function enqueueMonitorScrapeTarget(params: {
scrapeId,
20,
);
- expectedJobs.push(scrapeId);
}
- return { targetId: params.target.id, type: "scrape", expectedJobs };
+ return params.targetRun;
}
async function enqueueMonitorCrawlTarget(params: {
monitor: MonitorRow;
check: MonitorCheckRow;
target: MonitorTarget;
-}): Promise<{ targetId: string; type: "crawl"; crawlId: string }> {
+ targetRun: Extract<MonitorTargetRun, { type: "crawl" }>;
+}): Promise<Extract<MonitorTargetRun, { type: "crawl" }>> {
if (params.target.type !== "crawl") {
throw new Error("Expected crawl target");
}
- const crawlId = uuidv7();
+ const crawlId = params.targetRun.crawlId;
const body = crawlRequestSchema.parse({
url: params.target.url,
...(params.target.crawlOptions ?? {}),
@@ -729,7 +795,7 @@ async function enqueueMonitorCrawlTarget(params: {
uuidv7(),
);
- return { targetId: params.target.id, type: "crawl", crawlId };
+ return params.targetRun;
}
export async function processMonitorCheckJob(
@@ -770,19 +836,19 @@ export async function processMonitorCheckJob(
billing_status: lockId ? "reserved" : "not_applicable",
});
- const targetResults: unknown[] = [];
-
- for (const target of monitor.targets) {
- const result =
- target.type === "scrape"
- ? await enqueueMonitorScrapeTarget({ monitor, check, target })
- : await enqueueMonitorCrawlTarget({ monitor, check, target });
- targetResults.push(result);
- }
-
+ const targetResults = monitor.targets.map(createMonitorTargetRun);
await updateMonitorCheck(check.id, {
target_results: targetResults,
});
+
+ for (const [index, target] of monitor.targets.entries()) {
+ const targetRun = targetResults[index];
+ if (target.type === "scrape" && targetRun.type === "scrape") {
+ await enqueueMonitorScrapeTarget({ monitor, check, target, targetRun });
+ } else if (target.type === "crawl" && targetRun.type === "crawl") {
+ await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });
+ }
+ }
} catch (error) {
if (lockId) {
await autumnService.finalizeCreditsLock({
@@ -884,12 +950,18 @@ async function processRemovedPagesForCompletedCrawls(params: {
async function isMonitorCheckComplete(
check: MonitorCheckRow,
+ monitor?: MonitorRow,
): Promise<boolean> {
- const targetResults = Array.isArray(check.target_results)
+ let targetResults = Array.isArray(check.target_results)
? (check.target_results as any[])
: [];
- if (targetResults.length === 0) return false;
+ if (targetResults.length === 0) {
+ if (!monitor) return false;
+
+ targetResults = recoverScrapeTargetRunsFromMonitor(monitor) ?? [];
+ if (targetResults.length === 0) return false;
+ }
for (const target of targetResults) {
if (target?.type === "scrape") {
@@ -918,23 +990,13 @@ async function isMonitorCheckComplete(
return true;
}
-export function isMonitorCheckStale(
- check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at">,
- now: Date = new Date(),
-): boolean {
- const startedAt = check.started_at ?? check.updated_at ?? check.created_at;
- const startedAtMs = Date.parse(startedAt);
- if (!Number.isFinite(startedAtMs)) return false;
- return now.getTime() - startedAtMs >= MONITOR_CHECK_STALE_TIMEOUT_MS;
-}
-
async function failStaleMonitorCheck(params: {
monitor: MonitorRow;
check: MonitorCheckRow;
}): Promise<boolean> {
if (!isMonitorCheckStale(params.check)) return false;
- const error = "Monitor check exceeded the 24 hour running timeout.";
+ const error = MONITOR_CHECK_STALE_ERROR;
if (params.check.autumn_lock_id) {
await autumnService
.finalizeCreditsLock({
@@ -1041,9 +1103,12 @@ export async function reconcileRunningMonitorChecks(
if (await failStaleMonitorCheck({ monitor, check })) continue;
- const targetResults = Array.isArray(check.target_results)
+ let targetResults = Array.isArray(check.target_results)
? ([...check.target_results] as any[])
: [];
+ if (targetResults.length === 0) {
+ targetResults = recoverScrapeTargetRunsFromMonitor(monitor) ?? [];
+ }
await processRemovedPagesForCompletedCrawls({
monitor,
@@ -1052,10 +1117,13 @@ export async function reconcileRunningMonitorChecks(
});
if (
- !(await isMonitorCheckComplete({
- ...check,
- target_results: targetResults,
- }))
+ !(await isMonitorCheckComplete(
+ {
+ ...check,
+ target_results: targetResults,
+ },
+ monitor,
+ ))
) {
await updateMonitorCheck(check.id, { target_results: targetResults });
continue;
diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.ts
index c5ec32983..74a65709b 100644
--- a/apps/api/src/services/monitoring/scheduler.test.ts
+++ b/apps/api/src/services/monitoring/scheduler.test.ts
@@ -1,13 +1,16 @@
import { addMonitorCheckJob } from "./queue";
import { enqueueDueMonitorChecks } from "./scheduler";
+import { isMonitorCheckStale } from "./stale";
import {
advanceMonitorAfterSkippedCheck,
claimDueMonitors,
createMonitorCheck,
dispatchScheduledMonitorCheck,
+ getMonitorCheck,
updateMonitorCheck,
updateMonitorScheduleAfterRun,
} from "./store";
+import { autumnService } from "../autumn/autumn.service";
jest.mock("./queue", () => ({
addMonitorCheckJob: jest.fn(),
@@ -18,10 +21,23 @@ jest.mock("./store", () => ({
claimDueMonitors: jest.fn(),
createMonitorCheck: jest.fn(),
dispatchScheduledMonitorCheck: jest.fn(),
+ getMonitorCheck: jest.fn(),
updateMonitorCheck: jest.fn(),
updateMonitorScheduleAfterRun: jest.fn(),
}));
+jest.mock("./stale", () => ({
+ isMonitorCheckStale: jest.fn(),
+ MONITOR_CHECK_STALE_ERROR:
+ "Monitor check exceeded the 1 hour running timeout.",
+}));
+
+jest.mock("../autumn/autumn.service", () => ({
+ autumnService: {
+ finalizeCreditsLock: jest.fn(),
+ },
+}));
+
const mockAddMonitorCheckJob = addMonitorCheckJob as jest.MockedFunction<
typeof addMonitorCheckJob
>;
@@ -35,6 +51,16 @@ const mockDispatchScheduledMonitorCheck =
dispatchScheduledMonitorCheck as jest.MockedFunction<
typeof dispatchScheduledMonitorCheck
>;
+const mockGetMonitorCheck = getMonitorCheck as jest.MockedFunction<
+ typeof getMonitorCheck
+>;
+const mockIsMonitorCheckStale = isMonitorCheckStale as jest.MockedFunction<
+ typeof isMonitorCheckStale
+>;
+const mockFinalizeCreditsLock =
+ autumnService.finalizeCreditsLock as jest.MockedFunction<
+ typeof autumnService.finalizeCreditsLock
+ >;
const mockUpdateMonitorCheck = updateMonitorCheck as jest.MockedFunction<
typeof updateMonitorCheck
>;
@@ -64,6 +90,9 @@ describe("monitoring scheduler", () => {
mockAddMonitorCheckJob.mockResolvedValue(undefined);
mockAdvanceMonitorAfterSkippedCheck.mockResolvedValue(undefined);
mockUpdateMonitorScheduleAfterRun.mockResolvedValue(undefined);
+ mockGetMonitorCheck.mockResolvedValue(null);
+ mockIsMonitorCheckStale.mockReturnValue(false);
+ mockFinalizeCreditsLock.mockResolvedValue(undefined as any);
});
it("dispatches and advances a scheduled monitor before enqueueing its job", async () => {
@@ -131,4 +160,43 @@ describe("monitoring scheduler", () => {
check: skipped,
});
});
+
+ it("clears a stale current check before enqueueing a scheduled run", async () => {
+ const monitorWithCurrentCheck = {
+ ...monitor,
+ current_check_id: "stale-check",
+ } as any;
+ const staleCheck = { id: "stale-check", status: "running" } as any;
+ const failedStaleCheck = { ...staleCheck, status: "failed" } as any;
+ mockClaimDueMonitors.mockResolvedValue([monitorWithCurrentCheck]);
+ mockGetMonitorCheck.mockResolvedValue(staleCheck);
+ mockIsMonitorCheckStale.mockReturnValue(true);
+ mockUpdateMonitorCheck.mockResolvedValue(failedStaleCheck);
+
+ await expect(
+ enqueueDueMonitorChecks({ workerId: "worker-1" }),
+ ).resolves.toBe(1);
+
+ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(staleCheck.id, {
+ status: "failed",
+ finished_at: expect.any(String),
+ actual_credits: 0,
+ billing_status: "not_applicable",
+ error: "Monitor check exceeded the 1 hour running timeout.",
+ });
+ expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledWith({
+ monitor: monitorWithCurrentCheck,
+ check: failedStaleCheck,
+ });
+ expect(mockCreateMonitorCheck).toHaveBeenCalledWith({
+ monitor: { ...monitorWithCurrentCheck, current_check_id: null },
+ trigger: "scheduled",
+ scheduledFor: monitorWithCurrentCheck.next_run_at,
+ });
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({
+ monitorId: monitorWithCurrentCheck.id,
+ checkId: check.id,
+ teamId: monitorWithCurrentCheck.team_id,
+ });
+ });
});
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index 84a7984e7..61492e3b6 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -6,9 +6,13 @@ import {
claimDueMonitors,
createMonitorCheck,
dispatchScheduledMonitorCheck,
+ getMonitorCheck,
updateMonitorCheck,
updateMonitorScheduleAfterRun,
} from "./store";
+import { autumnService } from "../autumn/autumn.service";
+import { isMonitorCheckStale, MONITOR_CHECK_STALE_ERROR } from "./stale";
+import type { MonitorRow } from "./types";
const logger = _logger.child({ module: "monitoring-scheduler" });
@@ -35,10 +39,17 @@ export async function enqueueDueMonitorChecks(
});
let enqueued = 0;
- for (const monitor of monitors) {
+ for (let monitor of monitors) {
let check: Awaited<ReturnType<typeof createMonitorCheck>> | null = null;
let dispatched = false;
try {
+ if (monitor.current_check_id) {
+ const cleared = await clearFinishedOrStaleCurrentCheck(monitor);
+ if (cleared) {
+ monitor = { ...monitor, current_check_id: null };
+ }
+ }
+
if (monitor.current_check_id) {
const skipped = await createMonitorCheck({
monitor,
@@ -123,3 +134,60 @@ export async function enqueueDueMonitorChecks(
return enqueued;
}
+
+async function clearFinishedOrStaleCurrentCheck(
+ monitor: MonitorRow,
+): Promise<boolean> {
+ if (!monitor.current_check_id) return true;
+
+ const current = await getMonitorCheck(
+ monitor.team_id,
+ monitor.id,
+ monitor.current_check_id,
+ );
+ if (!current) return false;
+
+ if (current.status === "running" || current.status === "queued") {
+ if (!isMonitorCheckStale(current)) return false;
+
+ if (current.autumn_lock_id) {
+ await autumnService
+ .finalizeCreditsLock({
+ lockId: current.autumn_lock_id,
+ action: "release",
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: current.id,
+ },
+ })
+ .catch(error => {
+ logger.warn("Failed to release stale monitor check credit lock", {
+ error,
+ monitorId: monitor.id,
+ checkId: current.id,
+ lockId: current.autumn_lock_id,
+ });
+ });
+ }
+
+ const failed = await updateMonitorCheck(current.id, {
+ status: "failed",
+ finished_at: new Date().toISOString(),
+ actual_credits: 0,
+ billing_status: current.autumn_lock_id ? "released" : "not_applicable",
+ error: MONITOR_CHECK_STALE_ERROR,
+ });
+ await updateMonitorScheduleAfterRun({
+ monitor,
+ check: failed,
+ });
+ return true;
+ }
+
+ await updateMonitorScheduleAfterRun({
+ monitor,
+ check: current,
+ });
+ return true;
+}
diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.ts
new file mode 100644
index 000000000..669cf22dd
--- /dev/null
+++ b/apps/api/src/services/monitoring/stale.ts
@@ -0,0 +1,15 @@
+import type { MonitorCheckRow } from "./types";
+
+export const MONITOR_CHECK_STALE_TIMEOUT_MS = 60 * 60 * 1000;
+export const MONITOR_CHECK_STALE_ERROR =
+ "Monitor check exceeded the 1 hour running timeout.";
+
+export function isMonitorCheckStale(
+ check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at">,
+ now: Date = new Date(),
+): boolean {
+ const startedAt = check.started_at ?? check.updated_at ?? check.created_at;
+ const startedAtMs = Date.parse(startedAt);
+ if (!Number.isFinite(startedAtMs)) return false;
+ return now.getTime() - startedAtMs >= MONITOR_CHECK_STALE_TIMEOUT_MS;
+}
diff --git a/apps/api/src/services/webhook/delivery.test.ts b/apps/api/src/services/webhook/delivery.test.ts
new file mode 100644
index 000000000..845f7f930
--- /dev/null
+++ b/apps/api/src/services/webhook/delivery.test.ts
@@ -0,0 +1,30 @@
+import { WebhookEvent } from "./types";
+import { webhookEventMatchesFilter } from "./delivery";
+
+describe("webhook delivery", () => {
+ describe("webhookEventMatchesFilter", () => {
+ it("matches full monitor event names", () => {
+ expect(
+ webhookEventMatchesFilter(
+ ["monitor.page", "monitor.check.completed"],
+ WebhookEvent.MONITOR_PAGE,
+ ),
+ ).toBe(true);
+ expect(
+ webhookEventMatchesFilter(
+ ["monitor.page", "monitor.check.completed"],
+ WebhookEvent.MONITOR_CHECK_COMPLETED,
+ ),
+ ).toBe(true);
+ });
+
+ it("keeps legacy subtype filters for non-monitor webhooks", () => {
+ expect(webhookEventMatchesFilter(["page"], WebhookEvent.CRAWL_PAGE)).toBe(
+ true,
+ );
+ expect(
+ webhookEventMatchesFilter(["completed"], WebhookEvent.CRAWL_COMPLETED),
+ ).toBe(true);
+ });
+ });
+});
diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.ts
index fea161184..93c06e3ca 100644
--- a/apps/api/src/services/webhook/delivery.ts
+++ b/apps/api/src/services/webhook/delivery.ts
@@ -20,6 +20,31 @@ import { randomUUID } from "crypto";
const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";
const WEBHOOK_INSERT_BATCH_SIZE = 1000;
+type WebhookSendResult = {
+ attempted: boolean;
+ delivered?: boolean;
+ queued?: boolean;
+ skipped?: boolean;
+ statusCode?: number;
+};
+
+export function webhookEventMatchesFilter(
+ configuredEvents: string[] | undefined,
+ event: WebhookEvent,
+): boolean {
+ if (!configuredEvents?.length) {
+ return true;
+ }
+
+ const legacySubType = event.split(".")[1];
+ const namespaceSuffix = event.split(".").slice(1).join(".");
+ return (
+ configuredEvents.includes(event) ||
+ configuredEvents.includes(legacySubType) ||
+ configuredEvents.includes(namespaceSuffix)
+ );
+}
+
export class WebhookSender {
private config: WebhookConfig;
private secret?: string;
@@ -45,8 +70,10 @@ export class WebhookSender {
async send<T extends WebhookEvent>(
event: T,
data: WebhookEventDataMap[T],
- ): Promise<void> {
- if (!this.shouldSendEvent(event)) return;
+ ): Promise<WebhookSendResult> {
+ if (!this.shouldSendEvent(event)) {
+ return { attempted: false, skipped: true };
+ }
const payload = {
success: data.success,
@@ -64,10 +91,15 @@ export class WebhookSender {
);
if (data.awaitWebhook) {
- await delivery;
- } else {
- delivery.catch(() => {});
+ return { attempted: true, ...(await delivery) };
}
+
+ delivery.catch(() => {});
+ return {
+ attempted: true,
+ delivered: false,
+ queued: this.usesWebhookQueue(),
+ };
}
private shouldSendEvent(event: WebhookEvent): boolean {
@@ -75,24 +107,26 @@ export class WebhookSender {
return false;
}
- if (!this.config.events?.length) {
- return true;
- }
+ return webhookEventMatchesFilter(this.config.events, event);
+ }
- const subType = event.split(".")[1];
- return this.config.events.includes(subType as any);
+ private usesWebhookQueue(): boolean {
+ return Boolean(config.WEBHOOK_USE_RABBITMQ && config.NUQ_RABBITMQ_URL);
}
- private async deliver(payload: any, scrapeId?: string): Promise<void> {
+ private async deliver(
+ payload: any,
+ scrapeId?: string,
+ ): Promise<Omit<WebhookSendResult, "attempted">> {
const webhookHost = new URL(this.config.url).hostname;
if (isIPPrivate(webhookHost) && config.ALLOW_LOCAL_WEBHOOKS !== true) {
this.logger.warn("Aborting webhook call to private IP address", {
webhookUrl: this.config.url,
});
- return;
+ return { delivered: false, skipped: true };
}
- if (config.WEBHOOK_USE_RABBITMQ && config.NUQ_RABBITMQ_URL) {
+ if (this.usesWebhookQueue()) {
const queueMessage: WebhookQueueMessage = {
webhook_url: this.config.url,
payload,
@@ -118,7 +152,7 @@ export class WebhookSender {
throw error;
}
- return;
+ return { delivered: false, queued: true };
}
const payloadString = JSON.stringify(payload);
@@ -165,6 +199,7 @@ export class WebhookSender {
event: payload.type,
statusCode: res.status,
});
+ return { delivered: true, queued: false, statusCode: res.status };
} catch (error) {
this.logger.error("Failed to send webhook", {
error,
diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.ts
index 550c4f430..0ac175510 100644
--- a/apps/api/src/services/webhook/types.ts
+++ b/apps/api/src/services/webhook/types.ts
@@ -41,7 +41,7 @@ export type WebhookQueueMessage = {
webhookId: string;
id?: string;
jobId?: string;
- data: any[];
+ data: any;
error?: string;
metadata?: Record<string, string>;
};
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index e6ff468da..22707d59a 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -461,7 +461,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
monitoring: job.data.monitoring
- ? { ...job.data.monitoring, source: "discovered" }
+ ? {
+ ...job.data.monitoring,
+ source: "discovered" as const,
+ }
: undefined,
},
jobId,
@@ -585,9 +588,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
}
- await recordMonitorScrapeSuccess(job, doc).catch(error =>
- logger.warn("Failed to record monitor scrape result", { error }),
- );
+ await recordMonitorScrapeSuccess(job, doc);
logger.debug("Declaring job as done...");
await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
@@ -827,9 +828,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
}).catch(err => logger.warn("Scrape tracking failed", { error: err }));
- await recordMonitorScrapeFailure(job, error).catch(err =>
- logger.warn("Failed to record monitor scrape failure", { error: err }),
- );
+ await recordMonitorScrapeFailure(job, error);
return data;
} finally {
@@ -913,6 +912,9 @@ async function addKickoffSitemapJob(
webhook: sourceJob.data.webhook,
v1: sourceJob.data.v1,
apiKeyId: sourceJob.data.apiKeyId,
+ monitoring: sourceJob.data.monitoring
+ ? { ...sourceJob.data.monitoring, source: "discovered" as const }
+ : undefined,
} satisfies ScrapeJobKickoffSitemap,
jobId,
21,
@@ -968,6 +970,9 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
isCrawlSourceScrape: true,
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
+ monitoring: job.data.monitoring
+ ? { ...job.data.monitoring, source: "discovered" as const }
+ : undefined,
},
jobId,
await getJobPriority({ team_id: job.data.team_id, basePriority: 15 }),
@@ -1056,6 +1061,9 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
v1: job.data.v1,
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
+ monitoring: job.data.monitoring
+ ? { ...job.data.monitoring, source: "discovered" as const }
+ : undefined,
},
priority: jobPriority,
};
@@ -1170,6 +1178,9 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
zeroDataRetention:
job.data.zeroDataRetention || (sc.zeroDataRetention ?? false),
apiKeyId: job.data.apiKeyId,
+ monitoring: job.data.monitoring
+ ? { ...job.data.monitoring, source: "discovered" as const }
+ : undefined,
} satisfies ScrapeJobSingleUrls,
jobId: uuidv7(),
priority: jobPriority,