Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(pdf): by-reference deadline covers queue wait and scanned OCR rates (#4435)
Abimael Martell0be87dbfcf07
diff --git a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts b/apps/api/src/lib/__tests__/timeoutProcessing.test.tsindex c8502621d..7f98fa513 100644--- a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts+++ b/apps/api/src/lib/__tests__/timeoutProcessing.test.ts@@ -13,12 +13,12 @@ const T0 = 1_756_200_000_000;describe("composeTimeoutProcessing", () => {it("estimates from pages minus elapsed, rounded up to whole minutes", () => {- // 700 pages Γ 500ms + 60s base = 410s total; 2min elapsed β 290s- // remaining β rounds up to 5 minutes.+ // 700 pages Γ 1.25s + 60s base = 935s total; 2min elapsed β 815s+ // remaining β rounds up to 14 minutes; Retry-After caps at 10min.const { message, details } = composeTimeoutProcessing({pagesEstimate: 700,submittedAtMs: T0,- jobDeadlineAtMs: T0 + 11 * 60_000,+ jobDeadlineAtMs: T0 + 20 * 60_000,lastStatus: "running",nowMs: T0 + 2 * 60_000,});@@ -26,11 +26,11 @@ describe("composeTimeoutProcessing", () => {state: "processing_continues",documentPages: 700,jobStatus: "running",- estimatedRemainingSeconds: 300,- retryAfterSeconds: 300,+ estimatedRemainingSeconds: 840,+ retryAfterSeconds: 600,});expect(message).toContain("700-page PDF is still being processed");- expect(message).toContain("~5 minutes");+ expect(message).toContain("~14 minutes");});it("queued and published statuses read as queued, place is kept", () => {@@ -65,12 +65,12 @@ describe("composeTimeoutProcessing", () => {lastStatus: "running",nowMs: T0,});- expect(monster.details.estimatedRemainingSeconds).toBe(3_060);+ expect(monster.details.estimatedRemainingSeconds).toBe(7_560);expect(monster.details.retryAfterSeconds).toBe(600);});it("flags documents that may exceed the job's processing window", () => {- // 6000 pages β 51min of work against a 30-minute job deadline.+ // 6000 pages β 126min of work against a 30-minute job deadline.const { message, details } = composeTimeoutProcessing({pagesEstimate: 6_000,submittedAtMs: T0,@@ -101,7 +101,7 @@ describe("composeTimeoutProcessing β server estimate preference", () => {it("prefers fire-pdf's live estimate, aged since observation", () => {// Server said 9.5 minutes remaining, observed 90s ago β 8 minutes// after aging and minute-ceiling; the static formula (700 pages)- // would have said 5 minutes here.+ // would have said 14 minutes here.const { message, details } = composeTimeoutProcessing({pagesEstimate: 700,submittedAtMs: T0,@@ -131,8 +131,8 @@ describe("composeTimeoutProcessing β server estimate preference", () => {lastStatus: "running",nowMs: T0 + 2 * 60_000,});- // 700Γ500ms + 60s β 2min elapsed β 5 minutes (static path).- expect(withServer.details.estimatedRemainingSeconds).toBe(300);+ // 700Γ1.25s + 60s β 2min elapsed β 14 minutes (static path).+ expect(withServer.details.estimatedRemainingSeconds).toBe(840);});});diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex a2f6d2847..5d8a4b905 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -88,9 +88,11 @@ type ScrapeTimeoutProcessingDetails = {/** Matches BY_REFERENCE_DEADLINE_PER_PAGE_MS (fire-pdf/utils.ts) β the* conservative worst-case processing rate the job deadline is built- * from. Kept as a default parameter here so lib/error stays free of- * scraper imports. */-const PROCESSING_ESTIMATE_PER_PAGE_MS = 500;+ * from (covers fully scanned documents, not the text-extraction+ * median). Kept as a default parameter here so lib/error stays free of+ * scraper imports; keep the two constants in lockstep or the retry+ * hints understate what the deadline actually allows. */+const PROCESSING_ESTIMATE_PER_PAGE_MS = 1_250;/** Fixed overhead the estimate grants beyond pure page work: queue* pickup, render bootstrap, result assembly. */const PROCESSING_ESTIMATE_BASE_MS = 60_000;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex 609c40884..436e1674e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -1391,10 +1391,10 @@ describe("scrapePDFWithFirePDFAsync", () => {});};// Caller has only 60s left, but the JOB gets what the document- // needs: 5min base + 1000 pages Γ 500ms β 13.3min. The job then- // outlives this caller by design β cancel-on-abandon is skipped for- // by-reference β so the completion feeds the raw-sha cache and the- // adoption lookup for the customer's retry.+ // needs: 10min base + 1000 pages Γ 1.25s β 30min (capped). The job+ // then outlives this caller by design β cancel-on-abandon is+ // skipped for by-reference β so the completion feeds the raw-sha+ // cache and the adoption lookup for the customer's retry.const meta = makeMeta();meta.abort.scrapeTimeout = vi.fn(() => 60_000);@@ -1412,8 +1412,8 @@ describe("scrapePDFWithFirePDFAsync", () => {);const delta = new Date(submittedBody.deadline_at).getTime() - Date.now();- expect(delta).toBeGreaterThan(12 * 60 * 1_000);- expect(delta).toBeLessThanOrEqual(14 * 60 * 1_000);+ expect(delta).toBeGreaterThan(29 * 60 * 1_000);+ expect(delta).toBeLessThanOrEqual(30 * 60 * 1_000);});it("does NOT cancel a by-reference job when polling is abandoned", async () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.tsindex 356b2a456..ea413a987 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts@@ -153,21 +153,33 @@ describe("FirePDF async transport helpers", () => {});it("page-scales the by-reference job deadline independently of the caller", () => {- // 5min base + pages Γ 500ms, floored at the caller window, capped at+ // 10min base + pages Γ 1.25s, floored at the caller window, capped at// MAX_DEADLINE_MS. The caller's own polling stops at its window; the// decoupled job deadline is what lets the job finish server-side.- const FIVE_MIN = 5 * 60 * 1_000;+ // Base covers burst queue wait (measured 12-14min); per-page covers+ // the scanned worst case (~1.3s/page p90), not the text median.+ const TEN_MIN = 10 * 60 * 1_000;// Small doc, tiny caller window β base dominates.- expect(computeByReferenceDeadlineMs(60_000, 100)).toBe(FIVE_MIN + 50_000);+ expect(computeByReferenceDeadlineMs(60_000, 100)).toBe(TEN_MIN + 125_000);// Big doc β page term dominates, capped at 30 min.expect(computeByReferenceDeadlineMs(60_000, 6_543)).toBe(30 * 60 * 1_000);+ // The 931-page starvation case (2026-08-27): queue wait ate a+ // 12.8-min deadline down to 84s of processing. Now: 29.4min.+ expect(computeByReferenceDeadlineMs(60_000, 931)).toBe(+ TEN_MIN + 931 * 1_250,+ );+ // A scanned 798-pager needs ~17min of OCR; its budget now clears+ // that even before the queue-wait base is spent.+ expect(computeByReferenceDeadlineMs(60_000, 798)).toBe(+ TEN_MIN + 798 * 1_250,+ );// A caller with a LONGER explicit window than the page-scaled need// keeps its window (never advertise less than the caller has).- expect(computeByReferenceDeadlineMs(20 * 60 * 1_000, 100)).toBe(- 20 * 60 * 1_000,+ expect(computeByReferenceDeadlineMs(25 * 60 * 1_000, 100)).toBe(+ 25 * 60 * 1_000,);// No pages estimate β base + caller floor semantics still hold.- expect(computeByReferenceDeadlineMs(undefined, undefined)).toBe(FIVE_MIN);+ expect(computeByReferenceDeadlineMs(undefined, undefined)).toBe(TEN_MIN);});it("adds the shared FirePDF bearer credential when configured", () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex c80ffe58f..9318e0d84 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -53,13 +53,24 @@ export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {return Math.min(MAX_DEADLINE_MS, candidate);}-/** Rough worst-case processing rate for the page-scaled by-reference- * deadline. Prod xl-lane runs land well under this (a 6,543-page document- * processed in ~29 minutes β 270ms/page including queue wait); the slack- * absorbs queue depth without pushing every big document to the 30-min- * ceiling. */-const BY_REFERENCE_DEADLINE_PER_PAGE_MS = 500;-const BY_REFERENCE_DEADLINE_BASE_MS = 5 * 60 * 1_000;+/** Worst-case processing rate for the page-scaled by-reference deadline.+ * Must cover the SLOW class, not the median: scanned giants run full-page+ * OCR at ~1.3s/page p90 in prod (2026-08-27), and the original 500ms β+ * calibrated on a text-heavy 6,543-page run β starved them into deadline+ * fallback (a 798-page scan got 11.6min against ~17min of real OCR).+ *+ * The base must absorb QUEUE WAIT, which spends the same budget as+ * processing (the deadline clock starts at submit, not claim): measured+ * bursts queued giants 12-14 minutes behind five xl workers, so a 5-min+ * base guaranteed starvation regardless of the per-page term (a 931-page+ * doc reached its worker with 84s left and degraded 93% of its pages β+ * and that degraded result then poisoned content-adoption for every+ * retry). 10 min covers observed burst queues; genuinely fixing the+ * queue/processing conflation (claim-time budgets) is fire-pdf-side+ * follow-up work.+ */+const BY_REFERENCE_DEADLINE_PER_PAGE_MS = 1_250;+const BY_REFERENCE_DEADLINE_BASE_MS = 10 * 60 * 1_000;/*** Job deadline for by-reference submits, DECOUPLED from the caller's