Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
repair monitor crawl finalization
Developers Digest26f25a112383
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex aa8fcc120..7507f1618 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -8,6 +8,7 @@ import { derivePageIsMeaningful } from "./page-events";import {getMonitorForUpdate,getMonitorPage,+ ensureMonitorCheckCrawlTargetRun,hashMonitorUrl,insertMonitorCheckPages,upsertMonitorPage,@@ -113,6 +114,22 @@ export async function recordMonitorScrapeSuccess(const monitoring = job.data.monitoring;if (!monitoring || job.data.mode !== "single_urls") return;+ if (job.data.crawl_id) {+ await ensureMonitorCheckCrawlTargetRun({+ checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ crawlId: job.data.crawl_id,+ }).catch(error => {+ logger.warn("Failed to repair monitor crawl target run", {+ error,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ crawlId: job.data.crawl_id,+ });+ });+ }+const url = getDocumentUrl(doc, job.data.url);const previous = await getMonitorPage({monitorId: monitoring.monitorId,@@ -253,6 +270,22 @@ export async function recordMonitorScrapeFailure(const monitoring = job.data.monitoring;if (!monitoring || job.data.mode !== "single_urls") return;+ if (job.data.crawl_id) {+ await ensureMonitorCheckCrawlTargetRun({+ checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ crawlId: job.data.crawl_id,+ }).catch(repairError => {+ logger.warn("Failed to repair monitor crawl target run after failure", {+ error: repairError,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ crawlId: job.data.crawl_id,+ });+ });+ }+await insertMonitorCheckPages([{check_id: monitoring.checkId,diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 40716f540..d4cf892cd 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -113,21 +113,72 @@ function createMonitorTargetRun(target: MonitorTarget): MonitorTargetRun {};}-function recoverScrapeTargetRunsFromMonitor(- monitor: MonitorRow,-): MonitorTargetRun[] | null {- if (!monitor.targets.every(target => target.type === "scrape")) {- return null;+async function recoverTargetRunsFromRecordedPages(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+}): Promise<MonitorTargetRun[]> {+ const scrapeRuns: MonitorTargetRun[] = params.monitor.targets+ .filter((target): target is Extract<MonitorTarget, { type: "scrape" }> => {+ return target.type === "scrape";+ })+ .map(target => ({+ targetId: target.id,+ type: "scrape" as const,+ expectedJobs: target.urls.map(+ (_, index) => `recovered:${target.id}:${index}`,+ ),+ }));+ const crawlTargets = params.monitor.targets.filter(+ target => target.type === "crawl",+ );+ if (crawlTargets.length === 0) return scrapeRuns;++ const pages = await listMonitorCheckPages({+ teamId: params.monitor.team_id,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ limit: 100000,+ skip: 0,+ });+ const recovered = [...scrapeRuns];++ for (const target of crawlTargets) {+ const page = pages.find(+ candidate =>+ candidate.target_id === target.id &&+ typeof candidate.current_scrape_id === "string",+ );+ if (!page?.current_scrape_id) continue;++ const scrapeJob = await scrapeQueue+ .getJob(page.current_scrape_id, logger)+ .catch(error => {+ logger.warn(+ "Failed to recover monitor crawl target run from page job",+ {+ error,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ targetId: target.id,+ scrapeId: page.current_scrape_id,+ },+ );+ return null;+ });+ const crawlId =+ scrapeJob?.data?.mode === "single_urls"+ ? (scrapeJob.data.crawl_id ?? scrapeJob.groupId)+ : scrapeJob?.groupId;+ if (!crawlId) continue;++ recovered.push({+ targetId: target.id,+ type: "crawl",+ crawlId,+ });}- return monitor.targets.map(target => ({- targetId: target.id,- type: "scrape" as const,- expectedJobs:- target.type === "scrape"- ? target.urls.map((_, index) => `recovered:${target.id}:${index}`)- : [],- }));+ return recovered;}function withMonitorScrapeDefaults(@@ -1045,7 +1096,10 @@ async function isMonitorCheckComplete(if (targetResults.length === 0) {if (!monitor) return false;- targetResults = recoverScrapeTargetRunsFromMonitor(monitor) ?? [];+ targetResults = await recoverTargetRunsFromRecordedPages({+ monitor,+ check,+ });if (targetResults.length === 0) return false;}@@ -1188,7 +1242,49 @@ export async function reconcileRunningMonitorChecks(check.team_id,check.monitor_id,);- if (!monitor) continue;+ if (!monitor) {+ if (check.autumn_lock_id) {+ await autumnService+ .finalizeCreditsLock({+ lockId: check.autumn_lock_id,+ action: "release",+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: check.id,+ },+ })+ .catch(error => {+ logger.warn(+ "Failed to release orphaned monitor check credit lock",+ {+ error,+ monitorId: check.monitor_id,+ checkId: check.id,+ lockId: check.autumn_lock_id,+ },+ );+ });+ }++ await updateMonitorCheck(check.id, {+ status: "failed",+ finished_at: new Date().toISOString(),+ actual_credits: 0,+ billing_status: check.autumn_lock_id ? "released" : "not_applicable",+ error: "Monitor no longer exists.",+ });++ logger.warn("Failed orphaned monitor check", {+ monitorId: check.monitor_id,+ checkId: check.id,+ });+ continue;+ }++ if (check.status === "queued" && !isMonitorCheckStale(check)) {+ continue;+ }if (await failStaleMonitorCheck({ monitor, check })) continue;@@ -1196,7 +1292,10 @@ export async function reconcileRunningMonitorChecks(? ([...check.target_results] as any[]): [];if (targetResults.length === 0) {- targetResults = recoverScrapeTargetRunsFromMonitor(monitor) ?? [];+ targetResults = await recoverTargetRunsFromRecordedPages({+ monitor,+ check,+ });}await processRemovedPagesForCompletedCrawls({@@ -1214,7 +1313,9 @@ export async function reconcileRunningMonitorChecks(monitor,))) {- await updateMonitorCheck(check.id, { target_results: targetResults });+ if (targetResults.length > 0) {+ await updateMonitorCheck(check.id, { target_results: targetResults });+ }continue;}diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 2cec0f79b..1df8fe1bb 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -487,6 +487,54 @@ export async function updateMonitorCheck(return data as MonitorCheckRow;}+export async function ensureMonitorCheckCrawlTargetRun(params: {+ checkId: string;+ targetId: string;+ crawlId: string;+}): Promise<void> {+ const { data, error } = await supabase_service+ .from("monitor_checks")+ .select("target_results")+ .eq("id", params.checkId)+ .maybeSingle();++ throwIfError(error, "Failed to load monitor check target runs");+ if (!data) return;++ const targetResults = Array.isArray(data.target_results)+ ? data.target_results+ : [];+ const existing = targetResults.find(+ (target: any) =>+ target?.type === "crawl" &&+ target.targetId === params.targetId &&+ target.crawlId === params.crawlId,+ );+ if (existing) return;++ const withoutStaleTarget = targetResults.filter(+ (target: any) =>+ !(target?.type === "crawl" && target.targetId === params.targetId),+ );++ const { error: updateError } = await supabase_service+ .from("monitor_checks")+ .update({+ target_results: [+ ...withoutStaleTarget,+ {+ targetId: params.targetId,+ type: "crawl",+ crawlId: params.crawlId,+ },+ ],+ updated_at: new Date().toISOString(),+ })+ .eq("id", params.checkId);++ throwIfError(updateError, "Failed to repair monitor check target runs");+}+export async function insertMonitorCheckPages(pages: MonitorCheckPageInsert[],): Promise<void> {