Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(pdf): by-reference deadline covers queue wait and scanned OCR rates (#4435)

Abimael Martell0be87dbfcf07
diff --git a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts b/apps/api/src/lib/__tests__/timeoutProcessing.test.ts
index c8502621d..7f98fa513 100644
--- a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts
+++ b/apps/api/src/lib/__tests__/timeoutProcessing.test.ts
@@ -13,12 +13,12 @@ const T0 = 1_756_200_000_000;
describe("composeTimeoutProcessing", () => {
it("estimates from pages minus elapsed, rounded up to whole minutes", () => {
- // 700 pages Γ— 500ms + 60s base = 410s total; 2min elapsed β†’ 290s
- // remaining β†’ rounds up to 5 minutes.
+ // 700 pages Γ— 1.25s + 60s base = 935s total; 2min elapsed β†’ 815s
+ // remaining β†’ rounds up to 14 minutes; Retry-After caps at 10min.
const { message, details } = composeTimeoutProcessing({
pagesEstimate: 700,
submittedAtMs: T0,
- jobDeadlineAtMs: T0 + 11 * 60_000,
+ jobDeadlineAtMs: T0 + 20 * 60_000,
lastStatus: "running",
nowMs: T0 + 2 * 60_000,
});
@@ -26,11 +26,11 @@ describe("composeTimeoutProcessing", () => {
state: "processing_continues",
documentPages: 700,
jobStatus: "running",
- estimatedRemainingSeconds: 300,
- retryAfterSeconds: 300,
+ estimatedRemainingSeconds: 840,
+ retryAfterSeconds: 600,
});
expect(message).toContain("700-page PDF is still being processed");
- expect(message).toContain("~5 minutes");
+ expect(message).toContain("~14 minutes");
});
it("queued and published statuses read as queued, place is kept", () => {
@@ -65,12 +65,12 @@ describe("composeTimeoutProcessing", () => {
lastStatus: "running",
nowMs: T0,
});
- expect(monster.details.estimatedRemainingSeconds).toBe(3_060);
+ expect(monster.details.estimatedRemainingSeconds).toBe(7_560);
expect(monster.details.retryAfterSeconds).toBe(600);
});
it("flags documents that may exceed the job's processing window", () => {
- // 6000 pages β‰ˆ 51min of work against a 30-minute job deadline.
+ // 6000 pages β‰ˆ 126min of work against a 30-minute job deadline.
const { message, details } = composeTimeoutProcessing({
pagesEstimate: 6_000,
submittedAtMs: T0,
@@ -101,7 +101,7 @@ describe("composeTimeoutProcessing β€” server estimate preference", () => {
it("prefers fire-pdf's live estimate, aged since observation", () => {
// Server said 9.5 minutes remaining, observed 90s ago β†’ 8 minutes
// after aging and minute-ceiling; the static formula (700 pages)
- // would have said 5 minutes here.
+ // would have said 14 minutes here.
const { message, details } = composeTimeoutProcessing({
pagesEstimate: 700,
submittedAtMs: T0,
@@ -131,8 +131,8 @@ describe("composeTimeoutProcessing β€” server estimate preference", () => {
lastStatus: "running",
nowMs: T0 + 2 * 60_000,
});
- // 700Γ—500ms + 60s βˆ’ 2min elapsed β†’ 5 minutes (static path).
- expect(withServer.details.estimatedRemainingSeconds).toBe(300);
+ // 700Γ—1.25s + 60s βˆ’ 2min elapsed β†’ 14 minutes (static path).
+ expect(withServer.details.estimatedRemainingSeconds).toBe(840);
});
});
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index a2f6d2847..5d8a4b905 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -88,9 +88,11 @@ type ScrapeTimeoutProcessingDetails = {
/** Matches BY_REFERENCE_DEADLINE_PER_PAGE_MS (fire-pdf/utils.ts) β€” the
* conservative worst-case processing rate the job deadline is built
- * from. Kept as a default parameter here so lib/error stays free of
- * scraper imports. */
-const PROCESSING_ESTIMATE_PER_PAGE_MS = 500;
+ * from (covers fully scanned documents, not the text-extraction
+ * median). Kept as a default parameter here so lib/error stays free of
+ * scraper imports; keep the two constants in lockstep or the retry
+ * hints understate what the deadline actually allows. */
+const PROCESSING_ESTIMATE_PER_PAGE_MS = 1_250;
/** Fixed overhead the estimate grants beyond pure page work: queue
* pickup, render bootstrap, result assembly. */
const PROCESSING_ESTIMATE_BASE_MS = 60_000;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index 609c40884..436e1674e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -1391,10 +1391,10 @@ describe("scrapePDFWithFirePDFAsync", () => {
});
};
// Caller has only 60s left, but the JOB gets what the document
- // needs: 5min base + 1000 pages Γ— 500ms β‰ˆ 13.3min. The job then
- // outlives this caller by design β€” cancel-on-abandon is skipped for
- // by-reference β€” so the completion feeds the raw-sha cache and the
- // adoption lookup for the customer's retry.
+ // needs: 10min base + 1000 pages Γ— 1.25s β‰ˆ 30min (capped). The job
+ // then outlives this caller by design β€” cancel-on-abandon is
+ // skipped for by-reference β€” so the completion feeds the raw-sha
+ // cache and the adoption lookup for the customer's retry.
const meta = makeMeta();
meta.abort.scrapeTimeout = vi.fn(() => 60_000);
@@ -1412,8 +1412,8 @@ describe("scrapePDFWithFirePDFAsync", () => {
);
const delta = new Date(submittedBody.deadline_at).getTime() - Date.now();
- expect(delta).toBeGreaterThan(12 * 60 * 1_000);
- expect(delta).toBeLessThanOrEqual(14 * 60 * 1_000);
+ expect(delta).toBeGreaterThan(29 * 60 * 1_000);
+ expect(delta).toBeLessThanOrEqual(30 * 60 * 1_000);
});
it("does NOT cancel a by-reference job when polling is abandoned", async () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
index 356b2a456..ea413a987 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
@@ -153,21 +153,33 @@ describe("FirePDF async transport helpers", () => {
});
it("page-scales the by-reference job deadline independently of the caller", () => {
- // 5min base + pages Γ— 500ms, floored at the caller window, capped at
+ // 10min base + pages Γ— 1.25s, floored at the caller window, capped at
// MAX_DEADLINE_MS. The caller's own polling stops at its window; the
// decoupled job deadline is what lets the job finish server-side.
- const FIVE_MIN = 5 * 60 * 1_000;
+ // Base covers burst queue wait (measured 12-14min); per-page covers
+ // the scanned worst case (~1.3s/page p90), not the text median.
+ const TEN_MIN = 10 * 60 * 1_000;
// Small doc, tiny caller window β†’ base dominates.
- expect(computeByReferenceDeadlineMs(60_000, 100)).toBe(FIVE_MIN + 50_000);
+ expect(computeByReferenceDeadlineMs(60_000, 100)).toBe(TEN_MIN + 125_000);
// Big doc β†’ page term dominates, capped at 30 min.
expect(computeByReferenceDeadlineMs(60_000, 6_543)).toBe(30 * 60 * 1_000);
+ // The 931-page starvation case (2026-08-27): queue wait ate a
+ // 12.8-min deadline down to 84s of processing. Now: 29.4min.
+ expect(computeByReferenceDeadlineMs(60_000, 931)).toBe(
+ TEN_MIN + 931 * 1_250,
+ );
+ // A scanned 798-pager needs ~17min of OCR; its budget now clears
+ // that even before the queue-wait base is spent.
+ expect(computeByReferenceDeadlineMs(60_000, 798)).toBe(
+ TEN_MIN + 798 * 1_250,
+ );
// A caller with a LONGER explicit window than the page-scaled need
// keeps its window (never advertise less than the caller has).
- expect(computeByReferenceDeadlineMs(20 * 60 * 1_000, 100)).toBe(
- 20 * 60 * 1_000,
+ expect(computeByReferenceDeadlineMs(25 * 60 * 1_000, 100)).toBe(
+ 25 * 60 * 1_000,
);
// No pages estimate β†’ base + caller floor semantics still hold.
- expect(computeByReferenceDeadlineMs(undefined, undefined)).toBe(FIVE_MIN);
+ expect(computeByReferenceDeadlineMs(undefined, undefined)).toBe(TEN_MIN);
});
it("adds the shared FirePDF bearer credential when configured", () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
index c80ffe58f..9318e0d84 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -53,13 +53,24 @@ export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {
return Math.min(MAX_DEADLINE_MS, candidate);
}
-/** Rough worst-case processing rate for the page-scaled by-reference
- * deadline. Prod xl-lane runs land well under this (a 6,543-page document
- * processed in ~29 minutes β‰ˆ 270ms/page including queue wait); the slack
- * absorbs queue depth without pushing every big document to the 30-min
- * ceiling. */
-const BY_REFERENCE_DEADLINE_PER_PAGE_MS = 500;
-const BY_REFERENCE_DEADLINE_BASE_MS = 5 * 60 * 1_000;
+/** Worst-case processing rate for the page-scaled by-reference deadline.
+ * Must cover the SLOW class, not the median: scanned giants run full-page
+ * OCR at ~1.3s/page p90 in prod (2026-08-27), and the original 500ms β€”
+ * calibrated on a text-heavy 6,543-page run β€” starved them into deadline
+ * fallback (a 798-page scan got 11.6min against ~17min of real OCR).
+ *
+ * The base must absorb QUEUE WAIT, which spends the same budget as
+ * processing (the deadline clock starts at submit, not claim): measured
+ * bursts queued giants 12-14 minutes behind five xl workers, so a 5-min
+ * base guaranteed starvation regardless of the per-page term (a 931-page
+ * doc reached its worker with 84s left and degraded 93% of its pages β€”
+ * and that degraded result then poisoned content-adoption for every
+ * retry). 10 min covers observed burst queues; genuinely fixing the
+ * queue/processing conflation (claim-time budgets) is fire-pdf-side
+ * follow-up work.
+ */
+const BY_REFERENCE_DEADLINE_PER_PAGE_MS = 1_250;
+const BY_REFERENCE_DEADLINE_BASE_MS = 10 * 60 * 1_000;
/**
* Job deadline for by-reference submits, DECOUPLED from the caller's