Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(pdf): prefer fire-pdf's live remaining estimates in timeout messaging (#4417)
Abimael Martell73aa40b6a984
diff --git a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts b/apps/api/src/lib/__tests__/timeoutProcessing.test.tsindex 5694b93a6..c8502621d 100644--- a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts+++ b/apps/api/src/lib/__tests__/timeoutProcessing.test.ts@@ -97,6 +97,45 @@ describe("composeTimeoutProcessing", () => {});});+describe("composeTimeoutProcessing — server estimate preference", () => {+ it("prefers fire-pdf's live estimate, aged since observation", () => {+ // Server said 9.5 minutes remaining, observed 90s ago → 8 minutes+ // after aging and minute-ceiling; the static formula (700 pages)+ // would have said 5 minutes here.+ const { message, details } = composeTimeoutProcessing({+ pagesEstimate: 700,+ submittedAtMs: T0,+ lastStatus: "running",+ nowMs: T0 + 2 * 60_000,+ serverEstimate: { remainingMs: 9.5 * 60_000, observedAtMs: T0 + 30_000 },+ });+ expect(details.estimatedRemainingSeconds).toBe(8 * 60);+ expect(message).toContain("~8 minutes");+ });++ it("an aged-out server estimate floors at one minute", () => {+ const { details } = composeTimeoutProcessing({+ pagesEstimate: 700,+ submittedAtMs: T0,+ lastStatus: "running",+ nowMs: T0 + 20 * 60_000,+ serverEstimate: { remainingMs: 60_000, observedAtMs: T0 },+ });+ expect(details.estimatedRemainingSeconds).toBe(60);+ });++ it("falls back to static math when no server estimate was observed", () => {+ const withServer = composeTimeoutProcessing({+ pagesEstimate: 700,+ submittedAtMs: T0,+ lastStatus: "running",+ nowMs: T0 + 2 * 60_000,+ });+ // 700×500ms + 60s − 2min elapsed → 5 minutes (static path).+ expect(withServer.details.estimatedRemainingSeconds).toBe(300);+ });+});+describe("SCRAPE_TIMEOUT processing details transport", () => {it("survives the worker→controller serde round trip", () => {const { message, details } = composeTimeoutProcessing({diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex d8cbf3c34..a2f6d2847 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -109,6 +109,11 @@ export function composeTimeoutProcessing(args: {lastStatus: "queued" | "published" | "running";nowMs: number;perPageMs?: number;+ /** fire-pdf's live estimate from the last poll that carried one, and+ * when it was observed. Preferred over the static per-page math: the+ * server sees measured lane throughput and real queue depth, which+ * the static formula cannot. */+ serverEstimate?: { remainingMs: number; observedAtMs: number };}): { message: string; details: ScrapeTimeoutProcessingDetails } {const perPageMs = args.perPageMs ?? PROCESSING_ESTIMATE_PER_PAGE_MS;const pages = args.pagesEstimate;@@ -117,7 +122,18 @@ export function composeTimeoutProcessing(args: {? pages * perPageMs + PROCESSING_ESTIMATE_BASE_MS: 5 * 60_000;const elapsedMs = Math.max(0, args.nowMs - args.submittedAtMs);- const remainingMs = Math.max(60_000, totalMs - elapsedMs);+ // Live server estimate wins when available, aged by the time since we+ // observed it; the static formula is the fallback for older fire-pdf+ // builds and lanes without measured throughput.+ const serverRemainingMs =+ args.serverEstimate !== undefined+ ? args.serverEstimate.remainingMs -+ Math.max(0, args.nowMs - args.serverEstimate.observedAtMs)+ : undefined;+ const remainingMs =+ serverRemainingMs !== undefined+ ? Math.max(60_000, serverRemainingMs)+ : Math.max(60_000, totalMs - elapsedMs);const remainingMinutes = Math.ceil(remainingMs / 60_000);const estimatedRemainingSeconds = remainingMinutes * 60;const retryAfterSeconds = Math.min(600, estimatedRemainingSeconds);diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex a81471983..609c40884 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -1298,6 +1298,83 @@ describe("scrapePDFWithFirePDFAsync", () => {}});+ it("records fire-pdf's live estimate from 202 polls for timeout enrichment", async () => {+ const { fetchImpl } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 202,+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "xl" },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET",+ response: {+ status: 202,+ body: {+ scrape_id: "scrape-id-test",+ status: "running",+ estimated_remaining_ms: 240_000,+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ scrape_id: "scrape-id-test",+ status: "done",+ pages_processed: 500,+ },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: {+ schema_version: 1,+ markdown: "# ok",+ pages_processed: 500,+ failed_pages: null,+ partial_pages: null,+ },+ },+ },+ ]);++ const meta = makeMeta();+ let observed: unknown;+ const sleepSpy = async () => {+ // Snapshot mid-flight state after the first 202 recorded it —+ // completion clears the container, so assert before that.+ observed = { ...(meta.largePdfProcessing.current ?? {}) };+ };+ await scrapePDFWithFirePDFAsync(+ { ...meta, logger: meta.logger },+ { ...BY_REF },+ undefined,+ 500,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: sleepSpy },+ );++ // The last snapshot before completion carries the server estimate.+ expect(observed).toMatchObject({+ lastStatus: "running",+ serverEstimate: { remainingMs: 240_000 },+ });+ expect(+ (observed as { serverEstimate?: { observedAtMs: number } })+ .serverEstimate?.observedAtMs,+ ).toBeGreaterThan(0);+ });+it("advertises a page-scaled deadline decoupled from the caller window", async () => {let submittedBody: any;const fetchImpl: any = async (url: string, init: any) => {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 60eceb827..06994b70e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -276,9 +276,15 @@ export async function scrapePDFWithFirePDFAsync(sleep,now,random,- onNonTerminalStatus: status => {- if (meta.largePdfProcessing?.current) {- meta.largePdfProcessing.current.lastStatus = status;+ onNonTerminalStatus: (status, estimatedRemainingMs) => {+ const current = meta.largePdfProcessing?.current;+ if (!current) return;+ current.lastStatus = status;+ if (estimatedRemainingMs !== undefined) {+ current.serverEstimate = {+ remainingMs: estimatedRemainingMs,+ observedAtMs: now(),+ };}},});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.tsindex 59aec15c7..af6d93e95 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts@@ -25,8 +25,12 @@ type PollDeps = {random?: () => number;/** Observes each non-terminal status seen while polling — lets the* caller keep a live "where is this job" snapshot (used to enrich- * timeout errors for by-reference jobs that outlive the scrape). */- onNonTerminalStatus?: (status: "queued" | "published" | "running") => void;+ * timeout errors for by-reference jobs that outlive the scrape).+ * `estimatedRemainingMs` is fire-pdf's live estimate when present. */+ onNonTerminalStatus?: (+ status: "queued" | "published" | "running",+ estimatedRemainingMs?: number,+ ) => void;};type PollOk = { poll: PollResponse; pollCount: number };@@ -144,7 +148,10 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {parsed.data.status === "published" ||parsed.data.status === "running") {- deps.onNonTerminalStatus?.(parsed.data.status);+ deps.onNonTerminalStatus?.(+ parsed.data.status,+ parsed.data.estimated_remaining_ms,+ );}lastDelay = nextPollDelay(lastDelay, parsed.data.retry_after_ms, random);diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex 7e5bbf734..db5fb594d 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -41,6 +41,18 @@ export const pollResponseSchema = z.object({]),retry_after_ms: z.number().optional(),pages_processed: z.number().optional(),+ // Live remaining-time estimate computed server-side from lane+ // throughput + backlog (fire-pdf phase 2). Optional and additive;+ // absent on older fire-pdf builds or when the lane has no measured+ // throughput. `.catch(undefined)` so a malformed value (negative,+ // NaN, Infinity) degrades to "no estimate" instead of failing the+ // whole poll response.+ estimated_remaining_ms: z+ .number()+ .finite()+ .nonnegative()+ .optional()+ .catch(undefined),failed_pages: z.array(z.number()).nullable().optional(),partial_pages: z.array(z.number()).nullable().optional(),error_class: z.string().optional(),diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 421ba4096..4c0b0802b 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -187,6 +187,11 @@ export type Meta = {submittedAtMs: number;jobDeadlineAtMs?: number;lastStatus: "queued" | "published" | "running";+ /** fire-pdf's live remaining estimate from the last poll that+ * carried one, with its observation time — one atomic datum,+ * preferred over the static per-page math when composing the+ * timeout message. */+ serverEstimate?: { remainingMs: number; observedAtMs: number };};};documentPrefetch: