Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(pdf): prefer fire-pdf's live remaining estimates in timeout messaging (#4417)

Abimael Martell73aa40b6a984
diff --git a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts b/apps/api/src/lib/__tests__/timeoutProcessing.test.ts
index 5694b93a6..c8502621d 100644
--- a/apps/api/src/lib/__tests__/timeoutProcessing.test.ts
+++ b/apps/api/src/lib/__tests__/timeoutProcessing.test.ts
@@ -97,6 +97,45 @@ describe("composeTimeoutProcessing", () => {
});
});
+describe("composeTimeoutProcessing — server estimate preference", () => {
+ it("prefers fire-pdf's live estimate, aged since observation", () => {
+ // Server said 9.5 minutes remaining, observed 90s ago → 8 minutes
+ // after aging and minute-ceiling; the static formula (700 pages)
+ // would have said 5 minutes here.
+ const { message, details } = composeTimeoutProcessing({
+ pagesEstimate: 700,
+ submittedAtMs: T0,
+ lastStatus: "running",
+ nowMs: T0 + 2 * 60_000,
+ serverEstimate: { remainingMs: 9.5 * 60_000, observedAtMs: T0 + 30_000 },
+ });
+ expect(details.estimatedRemainingSeconds).toBe(8 * 60);
+ expect(message).toContain("~8 minutes");
+ });
+
+ it("an aged-out server estimate floors at one minute", () => {
+ const { details } = composeTimeoutProcessing({
+ pagesEstimate: 700,
+ submittedAtMs: T0,
+ lastStatus: "running",
+ nowMs: T0 + 20 * 60_000,
+ serverEstimate: { remainingMs: 60_000, observedAtMs: T0 },
+ });
+ expect(details.estimatedRemainingSeconds).toBe(60);
+ });
+
+ it("falls back to static math when no server estimate was observed", () => {
+ const withServer = composeTimeoutProcessing({
+ pagesEstimate: 700,
+ submittedAtMs: T0,
+ lastStatus: "running",
+ nowMs: T0 + 2 * 60_000,
+ });
+ // 700×500ms + 60s − 2min elapsed → 5 minutes (static path).
+ expect(withServer.details.estimatedRemainingSeconds).toBe(300);
+ });
+});
+
describe("SCRAPE_TIMEOUT processing details transport", () => {
it("survives the worker→controller serde round trip", () => {
const { message, details } = composeTimeoutProcessing({
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index d8cbf3c34..a2f6d2847 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -109,6 +109,11 @@ export function composeTimeoutProcessing(args: {
lastStatus: "queued" | "published" | "running";
nowMs: number;
perPageMs?: number;
+ /** fire-pdf's live estimate from the last poll that carried one, and
+ * when it was observed. Preferred over the static per-page math: the
+ * server sees measured lane throughput and real queue depth, which
+ * the static formula cannot. */
+ serverEstimate?: { remainingMs: number; observedAtMs: number };
}): { message: string; details: ScrapeTimeoutProcessingDetails } {
const perPageMs = args.perPageMs ?? PROCESSING_ESTIMATE_PER_PAGE_MS;
const pages = args.pagesEstimate;
@@ -117,7 +122,18 @@ export function composeTimeoutProcessing(args: {
? pages * perPageMs + PROCESSING_ESTIMATE_BASE_MS
: 5 * 60_000;
const elapsedMs = Math.max(0, args.nowMs - args.submittedAtMs);
- const remainingMs = Math.max(60_000, totalMs - elapsedMs);
+ // Live server estimate wins when available, aged by the time since we
+ // observed it; the static formula is the fallback for older fire-pdf
+ // builds and lanes without measured throughput.
+ const serverRemainingMs =
+ args.serverEstimate !== undefined
+ ? args.serverEstimate.remainingMs -
+ Math.max(0, args.nowMs - args.serverEstimate.observedAtMs)
+ : undefined;
+ const remainingMs =
+ serverRemainingMs !== undefined
+ ? Math.max(60_000, serverRemainingMs)
+ : Math.max(60_000, totalMs - elapsedMs);
const remainingMinutes = Math.ceil(remainingMs / 60_000);
const estimatedRemainingSeconds = remainingMinutes * 60;
const retryAfterSeconds = Math.min(600, estimatedRemainingSeconds);
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index a81471983..609c40884 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -1298,6 +1298,83 @@ describe("scrapePDFWithFirePDFAsync", () => {
}
});
+ it("records fire-pdf's live estimate from 202 polls for timeout enrichment", async () => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 202,
+ body: { scrape_id: "scrape-id-test", status: "queued", lane: "xl" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET",
+ response: {
+ status: 202,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "running",
+ estimated_remaining_ms: 240_000,
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ scrape_id: "scrape-id-test",
+ status: "done",
+ pages_processed: 500,
+ },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: {
+ schema_version: 1,
+ markdown: "# ok",
+ pages_processed: 500,
+ failed_pages: null,
+ partial_pages: null,
+ },
+ },
+ },
+ ]);
+
+ const meta = makeMeta();
+ let observed: unknown;
+ const sleepSpy = async () => {
+ // Snapshot mid-flight state after the first 202 recorded it —
+ // completion clears the container, so assert before that.
+ observed = { ...(meta.largePdfProcessing.current ?? {}) };
+ };
+ await scrapePDFWithFirePDFAsync(
+ { ...meta, logger: meta.logger },
+ { ...BY_REF },
+ undefined,
+ 500,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: sleepSpy },
+ );
+
+ // The last snapshot before completion carries the server estimate.
+ expect(observed).toMatchObject({
+ lastStatus: "running",
+ serverEstimate: { remainingMs: 240_000 },
+ });
+ expect(
+ (observed as { serverEstimate?: { observedAtMs: number } })
+ .serverEstimate?.observedAtMs,
+ ).toBeGreaterThan(0);
+ });
+
it("advertises a page-scaled deadline decoupled from the caller window", async () => {
let submittedBody: any;
const fetchImpl: any = async (url: string, init: any) => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 60eceb827..06994b70e 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -276,9 +276,15 @@ export async function scrapePDFWithFirePDFAsync(
sleep,
now,
random,
- onNonTerminalStatus: status => {
- if (meta.largePdfProcessing?.current) {
- meta.largePdfProcessing.current.lastStatus = status;
+ onNonTerminalStatus: (status, estimatedRemainingMs) => {
+ const current = meta.largePdfProcessing?.current;
+ if (!current) return;
+ current.lastStatus = status;
+ if (estimatedRemainingMs !== undefined) {
+ current.serverEstimate = {
+ remainingMs: estimatedRemainingMs,
+ observedAtMs: now(),
+ };
}
},
});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
index 59aec15c7..af6d93e95 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
@@ -25,8 +25,12 @@ type PollDeps = {
random?: () => number;
/** Observes each non-terminal status seen while polling — lets the
* caller keep a live "where is this job" snapshot (used to enrich
- * timeout errors for by-reference jobs that outlive the scrape). */
- onNonTerminalStatus?: (status: "queued" | "published" | "running") => void;
+ * timeout errors for by-reference jobs that outlive the scrape).
+ * `estimatedRemainingMs` is fire-pdf's live estimate when present. */
+ onNonTerminalStatus?: (
+ status: "queued" | "published" | "running",
+ estimatedRemainingMs?: number,
+ ) => void;
};
type PollOk = { poll: PollResponse; pollCount: number };
@@ -144,7 +148,10 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
parsed.data.status === "published" ||
parsed.data.status === "running"
) {
- deps.onNonTerminalStatus?.(parsed.data.status);
+ deps.onNonTerminalStatus?.(
+ parsed.data.status,
+ parsed.data.estimated_remaining_ms,
+ );
}
lastDelay = nextPollDelay(lastDelay, parsed.data.retry_after_ms, random);
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index 7e5bbf734..db5fb594d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -41,6 +41,18 @@ export const pollResponseSchema = z.object({
]),
retry_after_ms: z.number().optional(),
pages_processed: z.number().optional(),
+ // Live remaining-time estimate computed server-side from lane
+ // throughput + backlog (fire-pdf phase 2). Optional and additive;
+ // absent on older fire-pdf builds or when the lane has no measured
+ // throughput. `.catch(undefined)` so a malformed value (negative,
+ // NaN, Infinity) degrades to "no estimate" instead of failing the
+ // whole poll response.
+ estimated_remaining_ms: z
+ .number()
+ .finite()
+ .nonnegative()
+ .optional()
+ .catch(undefined),
failed_pages: z.array(z.number()).nullable().optional(),
partial_pages: z.array(z.number()).nullable().optional(),
error_class: z.string().optional(),
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 421ba4096..4c0b0802b 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -187,6 +187,11 @@ export type Meta = {
submittedAtMs: number;
jobDeadlineAtMs?: number;
lastStatus: "queued" | "published" | "running";
+ /** fire-pdf's live remaining estimate from the last poll that
+ * carried one, with its observation time — one atomic datum,
+ * preferred over the static per-page math when composing the
+ * timeout message. */
+ serverEstimate?: { remainingMs: number; observedAtMs: number };
};
};
documentPrefetch: