Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(monitoring): various fixes
mogery3afe6df1f48f
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex 4b1cc32ba..217925c92 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -8,7 +8,7 @@ describe("monitoring runner", () => {describe("isMonitorCheckStale", () => {const now = new Date("2026-05-06T12:00:00.000Z");- it("returns true when a running check is at least 24 hours old", () => {+ it("returns true when a running check is at least 1 hour old", () => {expect(isMonitorCheckStale({diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex e5371dafe..42a74a479 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -52,15 +52,65 @@ import type {import { withMarkdownFormat } from "./types";import { redisEvictConnection } from "../redis";import type { MonitorCheckJobData } from "./queue";+import {+ MONITOR_CHECK_STALE_ERROR,+ isMonitorCheckStale,+ MONITOR_CHECK_STALE_TIMEOUT_MS,+} from "./stale";const logger = _logger.child({ module: "monitoring-runner" });const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));-export const MONITOR_CHECK_STALE_TIMEOUT_MS = 24 * 60 * 60 * 1000;+export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };type PageResult = MonitorCheckPageInsert & {emailStatus?: string;};+type MonitorTargetRun =+ | {+ targetId: string;+ type: "scrape";+ expectedJobs: string[];+ }+ | {+ targetId: string;+ type: "crawl";+ crawlId: string;+ };++function createMonitorTargetRun(target: MonitorTarget): MonitorTargetRun {+ if (target.type === "scrape") {+ return {+ targetId: target.id,+ type: "scrape",+ expectedJobs: target.urls.map(() => uuidv7()),+ };+ }++ return {+ targetId: target.id,+ type: "crawl",+ crawlId: uuidv7(),+ };+}++function recoverScrapeTargetRunsFromMonitor(+ monitor: MonitorRow,+): MonitorTargetRun[] | null {+ if (!monitor.targets.every(target => target.type === "scrape")) {+ return null;+ }++ return monitor.targets.map(target => ({+ targetId: target.id,+ type: "scrape" as const,+ expectedJobs:+ target.type === "scrape"+ ? target.urls.map((_, index) => `recovered:${target.id}:${index}`)+ : [],+ }));+}+function withMonitorScrapeDefaults(options: Record<string, unknown>,): ScrapeOptions {@@ -548,12 +598,28 @@ async function sendNotifications(params: {webhook: params.monitor.webhook as any,v0: false,});- await sender?.send(WebhookEvent.MONITOR_CHECK_COMPLETED, {- success: params.check.status === "completed",- data: payload,- error: params.check.error ?? undefined,- });- webhookStatus = { attempted: true, success: true };+ try {+ const result = await sender?.send(WebhookEvent.MONITOR_CHECK_COMPLETED, {+ success: params.check.status === "completed",+ data: payload,+ error: params.check.error ?? undefined,+ awaitWebhook: true,+ });+ webhookStatus = {+ attempted: result?.attempted ?? false,+ success: result?.delivered === true,+ delivered: result?.delivered === true,+ queued: result?.queued === true,+ skipped: result?.skipped === true,+ };+ } catch (error) {+ webhookStatus = {+ attempted: true,+ success: false,+ delivered: false,+ error: error instanceof Error ? error.message : String(error),+ };+ }}const emailStatus = await sendMonitoringEmailSummary({@@ -579,14 +645,14 @@ async function enqueueMonitorScrapeTarget(params: {monitor: MonitorRow;check: MonitorCheckRow;target: MonitorTarget;-}): Promise<{ targetId: string; type: "scrape"; expectedJobs: string[] }> {+ targetRun: Extract<MonitorTargetRun, { type: "scrape" }>;+}): Promise<Extract<MonitorTargetRun, { type: "scrape" }>> {if (params.target.type !== "scrape") {throw new Error("Expected scrape target");}- const expectedJobs: string[] = [];- for (const url of params.target.urls) {- const scrapeId = uuidv7();+ for (const [index, url] of params.target.urls.entries()) {+ const scrapeId = params.targetRun.expectedJobs[index];const scrapeOptions = scrapeRequestSchema.parse({url,...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),@@ -632,22 +698,22 @@ async function enqueueMonitorScrapeTarget(params: {scrapeId,20,);- expectedJobs.push(scrapeId);}- return { targetId: params.target.id, type: "scrape", expectedJobs };+ return params.targetRun;}async function enqueueMonitorCrawlTarget(params: {monitor: MonitorRow;check: MonitorCheckRow;target: MonitorTarget;-}): Promise<{ targetId: string; type: "crawl"; crawlId: string }> {+ targetRun: Extract<MonitorTargetRun, { type: "crawl" }>;+}): Promise<Extract<MonitorTargetRun, { type: "crawl" }>> {if (params.target.type !== "crawl") {throw new Error("Expected crawl target");}- const crawlId = uuidv7();+ const crawlId = params.targetRun.crawlId;const body = crawlRequestSchema.parse({url: params.target.url,...(params.target.crawlOptions ?? {}),@@ -729,7 +795,7 @@ async function enqueueMonitorCrawlTarget(params: {uuidv7(),);- return { targetId: params.target.id, type: "crawl", crawlId };+ return params.targetRun;}export async function processMonitorCheckJob(@@ -770,19 +836,19 @@ export async function processMonitorCheckJob(billing_status: lockId ? "reserved" : "not_applicable",});- const targetResults: unknown[] = [];-- for (const target of monitor.targets) {- const result =- target.type === "scrape"- ? await enqueueMonitorScrapeTarget({ monitor, check, target })- : await enqueueMonitorCrawlTarget({ monitor, check, target });- targetResults.push(result);- }-+ const targetResults = monitor.targets.map(createMonitorTargetRun);await updateMonitorCheck(check.id, {target_results: targetResults,});++ for (const [index, target] of monitor.targets.entries()) {+ const targetRun = targetResults[index];+ if (target.type === "scrape" && targetRun.type === "scrape") {+ await enqueueMonitorScrapeTarget({ monitor, check, target, targetRun });+ } else if (target.type === "crawl" && targetRun.type === "crawl") {+ await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });+ }+ }} catch (error) {if (lockId) {await autumnService.finalizeCreditsLock({@@ -884,12 +950,18 @@ async function processRemovedPagesForCompletedCrawls(params: {async function isMonitorCheckComplete(check: MonitorCheckRow,+ monitor?: MonitorRow,): Promise<boolean> {- const targetResults = Array.isArray(check.target_results)+ let targetResults = Array.isArray(check.target_results)? (check.target_results as any[]): [];- if (targetResults.length === 0) return false;+ if (targetResults.length === 0) {+ if (!monitor) return false;++ targetResults = recoverScrapeTargetRunsFromMonitor(monitor) ?? [];+ if (targetResults.length === 0) return false;+ }for (const target of targetResults) {if (target?.type === "scrape") {@@ -918,23 +990,13 @@ async function isMonitorCheckComplete(return true;}-export function isMonitorCheckStale(- check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at">,- now: Date = new Date(),-): boolean {- const startedAt = check.started_at ?? check.updated_at ?? check.created_at;- const startedAtMs = Date.parse(startedAt);- if (!Number.isFinite(startedAtMs)) return false;- return now.getTime() - startedAtMs >= MONITOR_CHECK_STALE_TIMEOUT_MS;-}-async function failStaleMonitorCheck(params: {monitor: MonitorRow;check: MonitorCheckRow;}): Promise<boolean> {if (!isMonitorCheckStale(params.check)) return false;- const error = "Monitor check exceeded the 24 hour running timeout.";+ const error = MONITOR_CHECK_STALE_ERROR;if (params.check.autumn_lock_id) {await autumnService.finalizeCreditsLock({@@ -1041,9 +1103,12 @@ export async function reconcileRunningMonitorChecks(if (await failStaleMonitorCheck({ monitor, check })) continue;- const targetResults = Array.isArray(check.target_results)+ let targetResults = Array.isArray(check.target_results)? ([...check.target_results] as any[]): [];+ if (targetResults.length === 0) {+ targetResults = recoverScrapeTargetRunsFromMonitor(monitor) ?? [];+ }await processRemovedPagesForCompletedCrawls({monitor,@@ -1052,10 +1117,13 @@ export async function reconcileRunningMonitorChecks(});if (- !(await isMonitorCheckComplete({- ...check,- target_results: targetResults,- }))+ !(await isMonitorCheckComplete(+ {+ ...check,+ target_results: targetResults,+ },+ monitor,+ ))) {await updateMonitorCheck(check.id, { target_results: targetResults });continue;diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.tsindex c5ec32983..74a65709b 100644--- a/apps/api/src/services/monitoring/scheduler.test.ts+++ b/apps/api/src/services/monitoring/scheduler.test.ts@@ -1,13 +1,16 @@import { addMonitorCheckJob } from "./queue";import { enqueueDueMonitorChecks } from "./scheduler";+import { isMonitorCheckStale } from "./stale";import {advanceMonitorAfterSkippedCheck,claimDueMonitors,createMonitorCheck,dispatchScheduledMonitorCheck,+ getMonitorCheck,updateMonitorCheck,updateMonitorScheduleAfterRun,} from "./store";+import { autumnService } from "../autumn/autumn.service";jest.mock("./queue", () => ({addMonitorCheckJob: jest.fn(),@@ -18,10 +21,23 @@ jest.mock("./store", () => ({claimDueMonitors: jest.fn(),createMonitorCheck: jest.fn(),dispatchScheduledMonitorCheck: jest.fn(),+ getMonitorCheck: jest.fn(),updateMonitorCheck: jest.fn(),updateMonitorScheduleAfterRun: jest.fn(),}));+jest.mock("./stale", () => ({+ isMonitorCheckStale: jest.fn(),+ MONITOR_CHECK_STALE_ERROR:+ "Monitor check exceeded the 1 hour running timeout.",+}));++jest.mock("../autumn/autumn.service", () => ({+ autumnService: {+ finalizeCreditsLock: jest.fn(),+ },+}));+const mockAddMonitorCheckJob = addMonitorCheckJob as jest.MockedFunction<typeof addMonitorCheckJob>;@@ -35,6 +51,16 @@ const mockDispatchScheduledMonitorCheck =dispatchScheduledMonitorCheck as jest.MockedFunction<typeof dispatchScheduledMonitorCheck>;+const mockGetMonitorCheck = getMonitorCheck as jest.MockedFunction<+ typeof getMonitorCheck+>;+const mockIsMonitorCheckStale = isMonitorCheckStale as jest.MockedFunction<+ typeof isMonitorCheckStale+>;+const mockFinalizeCreditsLock =+ autumnService.finalizeCreditsLock as jest.MockedFunction<+ typeof autumnService.finalizeCreditsLock+ >;const mockUpdateMonitorCheck = updateMonitorCheck as jest.MockedFunction<typeof updateMonitorCheck>;@@ -64,6 +90,9 @@ describe("monitoring scheduler", () => {mockAddMonitorCheckJob.mockResolvedValue(undefined);mockAdvanceMonitorAfterSkippedCheck.mockResolvedValue(undefined);mockUpdateMonitorScheduleAfterRun.mockResolvedValue(undefined);+ mockGetMonitorCheck.mockResolvedValue(null);+ mockIsMonitorCheckStale.mockReturnValue(false);+ mockFinalizeCreditsLock.mockResolvedValue(undefined as any);});it("dispatches and advances a scheduled monitor before enqueueing its job", async () => {@@ -131,4 +160,43 @@ describe("monitoring scheduler", () => {check: skipped,});});++ it("clears a stale current check before enqueueing a scheduled run", async () => {+ const monitorWithCurrentCheck = {+ ...monitor,+ current_check_id: "stale-check",+ } as any;+ const staleCheck = { id: "stale-check", status: "running" } as any;+ const failedStaleCheck = { ...staleCheck, status: "failed" } as any;+ mockClaimDueMonitors.mockResolvedValue([monitorWithCurrentCheck]);+ mockGetMonitorCheck.mockResolvedValue(staleCheck);+ mockIsMonitorCheckStale.mockReturnValue(true);+ mockUpdateMonitorCheck.mockResolvedValue(failedStaleCheck);++ await expect(+ enqueueDueMonitorChecks({ workerId: "worker-1" }),+ ).resolves.toBe(1);++ expect(mockUpdateMonitorCheck).toHaveBeenCalledWith(staleCheck.id, {+ status: "failed",+ finished_at: expect.any(String),+ actual_credits: 0,+ billing_status: "not_applicable",+ error: "Monitor check exceeded the 1 hour running timeout.",+ });+ expect(mockUpdateMonitorScheduleAfterRun).toHaveBeenCalledWith({+ monitor: monitorWithCurrentCheck,+ check: failedStaleCheck,+ });+ expect(mockCreateMonitorCheck).toHaveBeenCalledWith({+ monitor: { ...monitorWithCurrentCheck, current_check_id: null },+ trigger: "scheduled",+ scheduledFor: monitorWithCurrentCheck.next_run_at,+ });+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({+ monitorId: monitorWithCurrentCheck.id,+ checkId: check.id,+ teamId: monitorWithCurrentCheck.team_id,+ });+ });});diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsindex 84a7984e7..61492e3b6 100644--- a/apps/api/src/services/monitoring/scheduler.ts+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -6,9 +6,13 @@ import {claimDueMonitors,createMonitorCheck,dispatchScheduledMonitorCheck,+ getMonitorCheck,updateMonitorCheck,updateMonitorScheduleAfterRun,} from "./store";+import { autumnService } from "../autumn/autumn.service";+import { isMonitorCheckStale, MONITOR_CHECK_STALE_ERROR } from "./stale";+import type { MonitorRow } from "./types";const logger = _logger.child({ module: "monitoring-scheduler" });@@ -35,10 +39,17 @@ export async function enqueueDueMonitorChecks(});let enqueued = 0;- for (const monitor of monitors) {+ for (let monitor of monitors) {let check: Awaited<ReturnType<typeof createMonitorCheck>> | null = null;let dispatched = false;try {+ if (monitor.current_check_id) {+ const cleared = await clearFinishedOrStaleCurrentCheck(monitor);+ if (cleared) {+ monitor = { ...monitor, current_check_id: null };+ }+ }+if (monitor.current_check_id) {const skipped = await createMonitorCheck({monitor,@@ -123,3 +134,60 @@ export async function enqueueDueMonitorChecks(return enqueued;}++async function clearFinishedOrStaleCurrentCheck(+ monitor: MonitorRow,+): Promise<boolean> {+ if (!monitor.current_check_id) return true;++ const current = await getMonitorCheck(+ monitor.team_id,+ monitor.id,+ monitor.current_check_id,+ );+ if (!current) return false;++ if (current.status === "running" || current.status === "queued") {+ if (!isMonitorCheckStale(current)) return false;++ if (current.autumn_lock_id) {+ await autumnService+ .finalizeCreditsLock({+ lockId: current.autumn_lock_id,+ action: "release",+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: current.id,+ },+ })+ .catch(error => {+ logger.warn("Failed to release stale monitor check credit lock", {+ error,+ monitorId: monitor.id,+ checkId: current.id,+ lockId: current.autumn_lock_id,+ });+ });+ }++ const failed = await updateMonitorCheck(current.id, {+ status: "failed",+ finished_at: new Date().toISOString(),+ actual_credits: 0,+ billing_status: current.autumn_lock_id ? "released" : "not_applicable",+ error: MONITOR_CHECK_STALE_ERROR,+ });+ await updateMonitorScheduleAfterRun({+ monitor,+ check: failed,+ });+ return true;+ }++ await updateMonitorScheduleAfterRun({+ monitor,+ check: current,+ });+ return true;+}diff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.tsnew file mode 100644index 000000000..669cf22dd--- /dev/null+++ b/apps/api/src/services/monitoring/stale.ts@@ -0,0 +1,15 @@+import type { MonitorCheckRow } from "./types";++export const MONITOR_CHECK_STALE_TIMEOUT_MS = 60 * 60 * 1000;+export const MONITOR_CHECK_STALE_ERROR =+ "Monitor check exceeded the 1 hour running timeout.";++export function isMonitorCheckStale(+ check: Pick<MonitorCheckRow, "started_at" | "updated_at" | "created_at">,+ now: Date = new Date(),+): boolean {+ const startedAt = check.started_at ?? check.updated_at ?? check.created_at;+ const startedAtMs = Date.parse(startedAt);+ if (!Number.isFinite(startedAtMs)) return false;+ return now.getTime() - startedAtMs >= MONITOR_CHECK_STALE_TIMEOUT_MS;+}diff --git a/apps/api/src/services/webhook/delivery.test.ts b/apps/api/src/services/webhook/delivery.test.tsnew file mode 100644index 000000000..845f7f930--- /dev/null+++ b/apps/api/src/services/webhook/delivery.test.ts@@ -0,0 +1,30 @@+import { WebhookEvent } from "./types";+import { webhookEventMatchesFilter } from "./delivery";++describe("webhook delivery", () => {+ describe("webhookEventMatchesFilter", () => {+ it("matches full monitor event names", () => {+ expect(+ webhookEventMatchesFilter(+ ["monitor.page", "monitor.check.completed"],+ WebhookEvent.MONITOR_PAGE,+ ),+ ).toBe(true);+ expect(+ webhookEventMatchesFilter(+ ["monitor.page", "monitor.check.completed"],+ WebhookEvent.MONITOR_CHECK_COMPLETED,+ ),+ ).toBe(true);+ });++ it("keeps legacy subtype filters for non-monitor webhooks", () => {+ expect(webhookEventMatchesFilter(["page"], WebhookEvent.CRAWL_PAGE)).toBe(+ true,+ );+ expect(+ webhookEventMatchesFilter(["completed"], WebhookEvent.CRAWL_COMPLETED),+ ).toBe(true);+ });+ });+});diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.tsindex fea161184..93c06e3ca 100644--- a/apps/api/src/services/webhook/delivery.ts+++ b/apps/api/src/services/webhook/delivery.ts@@ -20,6 +20,31 @@ import { randomUUID } from "crypto";const WEBHOOK_INSERT_QUEUE_KEY = "webhook-insert-queue";const WEBHOOK_INSERT_BATCH_SIZE = 1000;+type WebhookSendResult = {+ attempted: boolean;+ delivered?: boolean;+ queued?: boolean;+ skipped?: boolean;+ statusCode?: number;+};++export function webhookEventMatchesFilter(+ configuredEvents: string[] | undefined,+ event: WebhookEvent,+): boolean {+ if (!configuredEvents?.length) {+ return true;+ }++ const legacySubType = event.split(".")[1];+ const namespaceSuffix = event.split(".").slice(1).join(".");+ return (+ configuredEvents.includes(event) ||+ configuredEvents.includes(legacySubType) ||+ configuredEvents.includes(namespaceSuffix)+ );+}+export class WebhookSender {private config: WebhookConfig;private secret?: string;@@ -45,8 +70,10 @@ export class WebhookSender {async send<T extends WebhookEvent>(event: T,data: WebhookEventDataMap[T],- ): Promise<void> {- if (!this.shouldSendEvent(event)) return;+ ): Promise<WebhookSendResult> {+ if (!this.shouldSendEvent(event)) {+ return { attempted: false, skipped: true };+ }const payload = {success: data.success,@@ -64,10 +91,15 @@ export class WebhookSender {);if (data.awaitWebhook) {- await delivery;- } else {- delivery.catch(() => {});+ return { attempted: true, ...(await delivery) };}++ delivery.catch(() => {});+ return {+ attempted: true,+ delivered: false,+ queued: this.usesWebhookQueue(),+ };}private shouldSendEvent(event: WebhookEvent): boolean {@@ -75,24 +107,26 @@ export class WebhookSender {return false;}- if (!this.config.events?.length) {- return true;- }+ return webhookEventMatchesFilter(this.config.events, event);+ }- const subType = event.split(".")[1];- return this.config.events.includes(subType as any);+ private usesWebhookQueue(): boolean {+ return Boolean(config.WEBHOOK_USE_RABBITMQ && config.NUQ_RABBITMQ_URL);}- private async deliver(payload: any, scrapeId?: string): Promise<void> {+ private async deliver(+ payload: any,+ scrapeId?: string,+ ): Promise<Omit<WebhookSendResult, "attempted">> {const webhookHost = new URL(this.config.url).hostname;if (isIPPrivate(webhookHost) && config.ALLOW_LOCAL_WEBHOOKS !== true) {this.logger.warn("Aborting webhook call to private IP address", {webhookUrl: this.config.url,});- return;+ return { delivered: false, skipped: true };}- if (config.WEBHOOK_USE_RABBITMQ && config.NUQ_RABBITMQ_URL) {+ if (this.usesWebhookQueue()) {const queueMessage: WebhookQueueMessage = {webhook_url: this.config.url,payload,@@ -118,7 +152,7 @@ export class WebhookSender {throw error;}- return;+ return { delivered: false, queued: true };}const payloadString = JSON.stringify(payload);@@ -165,6 +199,7 @@ export class WebhookSender {event: payload.type,statusCode: res.status,});+ return { delivered: true, queued: false, statusCode: res.status };} catch (error) {this.logger.error("Failed to send webhook", {error,diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.tsindex 550c4f430..0ac175510 100644--- a/apps/api/src/services/webhook/types.ts+++ b/apps/api/src/services/webhook/types.ts@@ -41,7 +41,7 @@ export type WebhookQueueMessage = {webhookId: string;id?: string;jobId?: string;- data: any[];+ data: any;error?: string;metadata?: Record<string, string>;};diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex e6ff468da..22707d59a 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -461,7 +461,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,monitoring: job.data.monitoring- ? { ...job.data.monitoring, source: "discovered" }+ ? {+ ...job.data.monitoring,+ source: "discovered" as const,+ }: undefined,},jobId,@@ -585,9 +588,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}}- await recordMonitorScrapeSuccess(job, doc).catch(error =>- logger.warn("Failed to record monitor scrape result", { error }),- );+ await recordMonitorScrapeSuccess(job, doc);logger.debug("Declaring job as done...");await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);@@ -827,9 +828,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,}).catch(err => logger.warn("Scrape tracking failed", { error: err }));- await recordMonitorScrapeFailure(job, error).catch(err =>- logger.warn("Failed to record monitor scrape failure", { error: err }),- );+ await recordMonitorScrapeFailure(job, error);return data;} finally {@@ -913,6 +912,9 @@ async function addKickoffSitemapJob(webhook: sourceJob.data.webhook,v1: sourceJob.data.v1,apiKeyId: sourceJob.data.apiKeyId,+ monitoring: sourceJob.data.monitoring+ ? { ...sourceJob.data.monitoring, source: "discovered" as const }+ : undefined,} satisfies ScrapeJobKickoffSitemap,jobId,21,@@ -968,6 +970,9 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {isCrawlSourceScrape: true,zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,+ monitoring: job.data.monitoring+ ? { ...job.data.monitoring, source: "discovered" as const }+ : undefined,},jobId,await getJobPriority({ team_id: job.data.team_id, basePriority: 15 }),@@ -1056,6 +1061,9 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {v1: job.data.v1,zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,+ monitoring: job.data.monitoring+ ? { ...job.data.monitoring, source: "discovered" as const }+ : undefined,},priority: jobPriority,};@@ -1170,6 +1178,9 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {zeroDataRetention:job.data.zeroDataRetention || (sc.zeroDataRetention ?? false),apiKeyId: job.data.apiKeyId,+ monitoring: job.data.monitoring+ ? { ...job.data.monitoring, source: "discovered" as const }+ : undefined,} satisfies ScrapeJobSingleUrls,jobId: uuidv7(),priority: jobPriority,