Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Close finalize/reconciler races and drop dead inline scrape/crawl path
Developers Digestd8f775bfbf7d
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex fb1938985..c6e57179a 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -11,9 +11,7 @@ import {resolveNewGroupBackend,} from "../worker/nuq-router";import { ScrapeJobData } from "../../types";-import { getJobFromGCS } from "../../lib/gcs-jobs";import { includesFormat } from "../../lib/format-utils";-import { computeAndPersistPageDiff } from "./diff-orchestrator";import { normalizeMonitorFormats } from "./diff";import { autumnService } from "../autumn/autumn.service";import { getBillingQueue } from "../queue-service";@@ -39,9 +37,7 @@ import {calculateMonitorCheckActualCredits,getMonitorCheck,getMonitorForUpdate,- getMonitorPage,countMonitorCheckPages,- hashMonitorUrl,insertMonitorCheckPages,deleteMonitorCheckPages,listActiveMonitorPages,@@ -78,7 +74,6 @@ import {} from "./search/persist";const logger = _logger.child({ module: "monitoring-runner" });-const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;@@ -244,16 +239,6 @@ function withMonitorScrapeDefaults(};}-function getDocumentUrl(doc: any, fallback: string): string {- return doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? doc?.url ?? fallback;-}--function getDocumentStatusCode(doc: any): number | null {- return typeof doc?.metadata?.statusCode === "number"- ? doc.metadata.statusCode- : null;-}-export function estimateActualCredits(doc: any, options?: any): number {// Prefer the credits the scrape path actually recorded when present.const creditsUsed = doc?.metadata?.creditsUsed;@@ -271,67 +256,6 @@ export function estimateActualCredits(doc: any, options?: any): number {return 1;}-async function runSingleScrape(params: {- monitor: MonitorRow;- check: MonitorCheckRow;- target: MonitorTarget;- url: string;- requestId?: string;-}): Promise<{ scrapeId: string; doc: any; credits: number }> {- const scrapeId = uuidv7();- const scrapeOptions = scrapeRequestSchema.parse({- url: params.url,- ...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),- origin: "monitor",- });-- await logRequest({- id: scrapeId,- kind: "scrape",- api_version: "v2",- team_id: params.monitor.team_id,- origin: "monitor",- integration: null,- target_hint: params.url,- zeroDataRetention: false,- api_key_id: null,- });-- const internalOptions = {- teamId: params.monitor.team_id,- saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,- bypassBilling: true,- zeroDataRetention: false,- };- const job: NuQJob<ScrapeJobData> = {- id: scrapeId,- status: "active",- createdAt: new Date(),- priority: 20,- data: {- mode: "single_urls",- url: params.url,- team_id: params.monitor.team_id,- scrapeOptions,- internalOptions,- skipNuq: true,- origin: "monitor",- integration: null,- billing: { endpoint: "monitor", jobId: params.check.id },- requestId: params.requestId,- zeroDataRetention: false,- apiKeyId: null,- },- };-- const doc = await processJobInternal(job);- return {- scrapeId,- doc,- credits: estimateActualCredits(doc, params.target.scrapeOptions),- };-}-// Deep-mode search-monitor page scrape, inline (skipNuq) so it bypasses scrape// concurrency; caller bounds fan-out via SEARCH_SCRAPE_CONCURRENCY. Never billed// per-page — search monitors bill flat at the check level.@@ -401,339 +325,6 @@ async function scrapeSearchMonitorPage(params: {};}-async function diffAndPersistPage(params: {- monitor: MonitorRow;- check: MonitorCheckRow;- target: MonitorTarget;- url: string;- scrapeId: string;- doc: any;- source: "explicit" | "discovered";- creditsUsed?: number;-}): Promise<PageResult> {- const previous = await getMonitorPage({- monitorId: params.monitor.id,- targetId: params.target.id,- url: params.url,- });-- const ctFormat = Array.isArray(params.target.scrapeOptions?.formats)- ? (params.target.scrapeOptions!.formats as any[]).find(- (f: any) => f?.type === "changeTracking",- )- : undefined;- const { status, diffGcsKey, diffTextBytes, diffJsonBytes, judgment, error } =- await computeAndPersistPageDiff({- teamId: params.monitor.team_id,- monitorId: params.monitor.id,- checkId: params.check.id,- url: params.url,- scrapeId: params.scrapeId,- doc: params.doc,- previous: previous- ? {- last_scrape_id: previous.last_scrape_id,- is_removed: previous.is_removed,- }- : null,- formats: params.target.scrapeOptions?.formats,- goal: params.monitor.judge_enabled ? params.monitor.goal : null,- extractionPrompt: ctFormat?.prompt ?? null,- });-- await upsertMonitorPage({- monitorId: params.monitor.id,- teamId: params.monitor.team_id,- targetId: params.target.id,- url: params.url,- source: params.source,- checkId: params.check.id,- scrapeId: params.scrapeId,- status,- metadata: {- title: params.doc?.metadata?.title ?? null,- statusCode: getDocumentStatusCode(params.doc),- contentType: params.doc?.metadata?.contentType ?? null,- numPages: params.doc?.metadata?.numPages ?? null,- proxyUsed: params.doc?.metadata?.proxyUsed ?? null,- postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,- creditsUsed: params.creditsUsed ?? null,- },- });-- return {- check_id: params.check.id,- monitor_id: params.monitor.id,- team_id: params.monitor.team_id,- target_id: params.target.id,- url: params.url,- url_hash: hashMonitorUrl(params.url),- status,- previous_scrape_id: previous?.last_scrape_id ?? null,- current_scrape_id: params.scrapeId,- diff_gcs_key: diffGcsKey,- diff_text_bytes: diffTextBytes,- diff_json_bytes: diffJsonBytes,- status_code: getDocumentStatusCode(params.doc),- ...(error ? { error } : {}),- metadata: {- title: params.doc?.metadata?.title ?? null,- contentType: params.doc?.metadata?.contentType ?? null,- numPages: params.doc?.metadata?.numPages ?? null,- proxyUsed: params.doc?.metadata?.proxyUsed ?? null,- postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,- creditsUsed: params.creditsUsed ?? null,- },- judgment,- emailStatus: status,- };-}--async function runScrapeTarget(params: {- monitor: MonitorRow;- check: MonitorCheckRow;- target: MonitorTarget;-}): Promise<{ pages: PageResult[]; credits: number; targetResult: any }> {- if (params.target.type !== "scrape") {- return { pages: [], credits: 0, targetResult: null };- }-- const pages: PageResult[] = [];- let credits = 0;-- for (const url of params.target.urls) {- try {- const result = await runSingleScrape({- monitor: params.monitor,- check: params.check,- target: params.target,- url,- });- credits += result.credits;- pages.push(- await diffAndPersistPage({- monitor: params.monitor,- check: params.check,- target: params.target,- url,- scrapeId: result.scrapeId,- doc: result.doc,- source: "explicit",- creditsUsed: result.credits,- }),- );- } catch (error) {- pages.push({- check_id: params.check.id,- monitor_id: params.monitor.id,- team_id: params.monitor.team_id,- target_id: params.target.id,- url,- url_hash: hashMonitorUrl(url),- status: "error",- error: error instanceof Error ? error.message : String(error),- emailStatus: "error",- });- }- }-- return {- pages,- credits,- targetResult: {- targetId: params.target.id,- type: params.target.type,- pages: pages.length,- credits,- },- };-}--async function runCrawlTarget(params: {- monitor: MonitorRow;- check: MonitorCheckRow;- target: MonitorTarget;-}): Promise<{ pages: PageResult[]; credits: number; targetResult: any }> {- if (params.target.type !== "crawl") {- return { pages: [], credits: 0, targetResult: null };- }-- const crawlId = uuidv7();- const body = crawlRequestSchema.parse({- url: params.target.url,- ...(params.target.crawlOptions ?? {}),- scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),- origin: "monitor",- }) as CrawlRequest;-- await logRequest({- id: crawlId,- kind: "crawl",- api_version: "v2",- team_id: params.monitor.team_id,- origin: "monitor",- integration: null,- target_hint: body.url,- zeroDataRetention: false,- api_key_id: null,- });-- const crawlerOptions = {- ...body,- url: undefined,- scrapeOptions: undefined,- prompt: undefined,- };-- const sc: StoredCrawl = {- originUrl: body.url,- crawlerOptions: toV0CrawlerOptions(crawlerOptions),- scrapeOptions: body.scrapeOptions,- internalOptions: {- disableSmartWaitCache: true,- teamId: params.monitor.team_id,- saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,- zeroDataRetention: false,- bypassBilling: true,- },- team_id: params.monitor.team_id,- createdAt: Date.now(),- maxConcurrency: body.maxConcurrency,- zeroDataRetention: false,- };-- const crawler = crawlToCrawler(crawlId, sc, null);- try {- sc.robots = await crawler.getRobotsTxt(- body.scrapeOptions.skipTlsVerification,- );- } catch {- // Non-fatal robots fetch failure, same as the public crawl controller.- }-- sc.queueBackend = await resolveNewGroupBackend(sc.team_id);- await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000, {- backend: sc.queueBackend,- maxConcurrency: sc.maxConcurrency,- delaySeconds: sc.crawlerOptions?.delay,- });- await saveCrawl(crawlId, sc);- await markCrawlActive(crawlId);-- await _addScrapeJobToBullMQ(- {- url: body.url,- mode: "kickoff",- team_id: params.monitor.team_id,- crawlerOptions,- scrapeOptions: sc.scrapeOptions,- internalOptions: sc.internalOptions,- origin: "monitor",- integration: null,- billing: { endpoint: "monitor", jobId: params.check.id },- crawl_id: crawlId,- v1: true,- zeroDataRetention: false,- apiKeyId: null,- },- uuidv7(),- );-- const started = Date.now();- let status = "scraping";- let total = 0;- while (Date.now() - started < 30 * 60 * 1000) {- const group = await crawlGroup.getGroup(crawlId);- const stats = await scrapeQueue.getGroupNumericStats(crawlId, logger);- status = group?.status ?? "scraping";- total =- (stats.completed ?? 0) +- (stats.active ?? 0) +- (stats.queued ?? 0) +- (stats.backlog ?? 0);- if (status !== "active" && status !== "scraping") break;- await poll(1000);- }-- const doneJobs = await scrapeQueue.getCrawlJobsForListing(- crawlId,- Math.max(total, 1),- 0,- logger,- );-- const pages: PageResult[] = [];- const seen = new Set<string>();- let credits = 0;-- for (const job of doneJobs) {- const doc = job.returnvalue ?? (await getJobFromGCS(job.id))?.[0];- if (!doc) continue;- const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);- seen.add(hashMonitorUrl(url).toString("hex"));- const pageCredits = estimateActualCredits(doc, body.scrapeOptions);- credits += pageCredits;- pages.push(- await diffAndPersistPage({- monitor: params.monitor,- check: params.check,- target: params.target,- url,- scrapeId: job.id,- doc,- source: "discovered",- creditsUsed: pageCredits,- }),- );- }-- if (status === "completed") {- const previousPages = await listActiveMonitorPages({- monitorId: params.monitor.id,- targetId: params.target.id,- });- for (const previous of previousPages) {- if (seen.has(previous.url_hash.toString("hex"))) continue;- await upsertMonitorPage({- monitorId: params.monitor.id,- teamId: params.monitor.team_id,- targetId: params.target.id,- url: previous.url,- source: previous.source,- checkId: params.check.id,- scrapeId: previous.last_scrape_id,- status: "removed",- metadata: previous.metadata,- });- pages.push({- check_id: params.check.id,- monitor_id: params.monitor.id,- team_id: params.monitor.team_id,- target_id: params.target.id,- url: previous.url,- url_hash: previous.url_hash,- status: "removed",- previous_scrape_id: previous.last_scrape_id,- current_scrape_id: null,- emailStatus: "removed",- });- }- }-- return {- pages,- credits,- targetResult: {- targetId: params.target.id,- type: params.target.type,- crawlId,- status,- pages: pages.length,- credits,- },- };-}-function summarize(pages: PageResult[]) {return {totalPages: pages.length,@@ -1396,6 +987,23 @@ export async function processMonitorCheckJob(target_results: targetResults,});} catch (error) {+ // Atomically flip running -> failed. Returns null when the check already+ // reached a terminal status — i.e. the reconciler finalized it (completed,+ // billed, lock confirmed) before this late catch ran. In that case we must+ // not clobber its terminal state or release its now-confirmed credit lock;+ // the reconciler already owns billing, notifications, and scheduling.+ const failed = await updateMonitorCheckIfRunning(check.id, {+ status: "failed",+ finished_at: new Date().toISOString(),+ billing_status: lockId ? "released" : "failed",+ error: error instanceof Error ? error.message : String(error),+ });++ if (!failed) {+ throw error;+ }+ check = failed;+if (lockId) {await autumnService.finalizeCreditsLock({lockId,@@ -1408,13 +1016,6 @@ export async function processMonitorCheckJob(});}- check = await updateMonitorCheck(check.id, {- status: "failed",- finished_at: new Date().toISOString(),- billing_status: lockId ? "released" : "failed",- error: error instanceof Error ? error.message : String(error),- });-if (await claimMonitorNotification(check.id).catch(error => {logger.warn(@@ -1724,10 +1325,16 @@ export async function reconcileRunningMonitorChecks(if (await failStaleMonitorCheck({ monitor, check })) continue;+ // Snapshot from listRunningMonitorChecks; may be stale relative to the+ // inline handler that is still writing this check.let targetResults = Array.isArray(check.target_results)? ([...check.target_results] as any[]): [];- if (targetResults.length === 0) {+ // True only when the persisted snapshot was empty and we rebuild the target+ // runs from recorded pages. That is the only case it is safe to write back+ // below: there is no live target_results to overwrite.+ const recoveredFromEmpty = targetResults.length === 0;+ if (recoveredFromEmpty) {targetResults = await recoverTargetRunsFromRecordedPages({monitor,check,@@ -1749,7 +1356,13 @@ export async function reconcileRunningMonitorChecks(monitor,))) {- if (targetResults.length > 0) {+ // Only persist target_results we recovered from an empty snapshot. Writing+ // back a non-empty stale snapshot here can DOWNGRADE a searchCompleted=true+ // that the inline handler persisted after this reconciler loaded its+ // snapshot, reverting the marker and stranding the check until the stale+ // reaper. The complete-path write below is safe: a search target can only+ // be complete once its snapshot already carries searchCompleted=true.+ if (recoveredFromEmpty && targetResults.length > 0) {await updateMonitorCheck(check.id, { target_results: targetResults });}continue;