Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

perf(api): poll fire-pdf jobs on a page-aware early schedule (#4880)

Abimael Martell463213eb8fe6
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
index e29f8dfdf..4dbe60fcb 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
@@ -158,6 +158,60 @@ describe("scrapePDFWithFirePDFAsync β€” deadline and submit lifecycle", () => {
expect(pollTimes).toEqual([1000, 3000, 7000, 12000, 17000, 21000, 22000]);
});
+ it("polls on the page-aware early schedule when the caller passes a page estimate", async () => {
+ let virtualNow = 1_000_000;
+ const sleeps: number[] = [];
+ let polls = 0;
+ const fetchImpl: any = async (url: string, init: any) => {
+ const method = (init?.method ?? "GET").toUpperCase();
+ if (/\/jobs$/.test(url) && method === "POST") {
+ return jsonResp({
+ status: 202,
+ body: { scrape_id: "x", status: "queued", lane: "standard" },
+ });
+ }
+ if (/\/jobs\/scrape-id-test$/.test(url)) {
+ polls++;
+ return jsonResp(
+ polls <= 3
+ ? { status: 202, body: { scrape_id: "x", status: "running" } }
+ : {
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 3 },
+ },
+ );
+ }
+ return jsonResp({
+ status: 200,
+ body: { markdown: "ok", pages_processed: 3 },
+ });
+ };
+
+ // Same flow as above but with a 3-page estimate (inline and
+ // by-reference callers both pass it): first poll at the 3-page p50
+ // rather than the 1s floor, then the fast interval.
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ 3,
+ undefined,
+ {
+ fetchImpl,
+ fallbackImpl: vi.fn(),
+ sleepImpl: async ms => {
+ sleeps.push(ms);
+ virtualNow += ms;
+ },
+ nowImpl: () => virtualNow,
+ randomImpl: () => 0,
+ },
+ );
+
+ expect(result.markdown).toBe("ok");
+ expect(sleeps).toEqual([1200, 300, 300, 300]);
+ });
+
it("retries the submit once when a closing api pod answers with Fastify's 503 body", async () => {
const before = await counterValue(firePdfAsyncSubmitRetriesTotal, {
trigger: "http_503_closing",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFEarlyPoll.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFEarlyPoll.test.ts
new file mode 100644
index 000000000..e66bc78f2
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFEarlyPoll.test.ts
@@ -0,0 +1,215 @@
+import { earlyPollDelay } from "../fire-pdf/early-poll";
+import { pollUntilTerminal } from "../fire-pdf/poll";
+import { jsonResp, makeMeta } from "./firePDFAsyncFixtures";
+
+const T0 = 1_000_000;
+
+// Drives pollUntilTerminal on a virtual clock. The job is `running` until
+// `doneAtMs` of virtual time has passed (or `doneAfterPolls` polls were
+// served), and `runningBody` is what every running poll answers with.
+async function runPoll(opts: {
+ pagesEstimate?: number;
+ initialDelay?: number;
+ doneAtMs?: number;
+ doneAfterPolls?: number;
+ jobDeadlineAtMs?: number;
+ runningBody?: Record<string, unknown>;
+}) {
+ let virtualNow = T0;
+ const sleeps: number[] = [];
+ let polls = 0;
+ const fetchImpl: any = async () => {
+ polls++;
+ const done =
+ (opts.doneAtMs !== undefined && virtualNow - T0 >= opts.doneAtMs) ||
+ (opts.doneAfterPolls !== undefined && polls >= opts.doneAfterPolls);
+ return done
+ ? jsonResp({
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 1 },
+ })
+ : jsonResp({
+ status: 202,
+ body: {
+ scrape_id: "x",
+ status: "running",
+ ...opts.runningBody,
+ },
+ });
+ };
+ const result = await pollUntilTerminal({
+ baseUrl: "http://fire-pdf.test",
+ scrapeId: "x",
+ initialDelay: opts.initialDelay,
+ pagesEstimate: opts.pagesEstimate,
+ pollingDeadline: T0 + 10 * 60_000,
+ meta: makeMeta(),
+ fetchImpl,
+ sleep: async ms => {
+ sleeps.push(ms);
+ virtualNow += ms;
+ },
+ now: () => virtualNow,
+ random: () => 0,
+ jobDeadlineAtMs:
+ opts.jobDeadlineAtMs !== undefined
+ ? T0 + opts.jobDeadlineAtMs
+ : undefined,
+ });
+ return { sleeps, pollCount: result.pollCount };
+}
+
+const repeat = (ms: number, n: number) => Array<number>(n).fill(ms);
+
+describe("pollUntilTerminal β€” page-aware early polling", () => {
+ it("polls a small document at its expected finish time, then every ~300ms through the slow tail", async () => {
+ // 3 pages: first poll at 1.2s, fast polls while under 5s (the 13th
+ // lands at 5.1s), then the regular backoff from its 1s floor.
+ const { sleeps, pollCount } = await runPoll({
+ pagesEstimate: 3,
+ doneAtMs: 6_000,
+ });
+ expect(sleeps).toEqual([1200, ...repeat(300, 13), 1000]);
+ expect(pollCount).toBe(15);
+ });
+
+ it("sees a job that finishes at the expected time on the very first poll", async () => {
+ const { sleeps, pollCount } = await runPoll({
+ pagesEstimate: 1,
+ doneAtMs: 1_000,
+ });
+ expect(sleeps).toEqual([1000]);
+ expect(pollCount).toBe(1);
+ });
+
+ it("falls back to the doubling backoff, capped at 5s, once past the slow tail", async () => {
+ const { sleeps } = await runPoll({
+ pagesEstimate: 3,
+ doneAtMs: 30_000,
+ });
+ const afterTail = sleeps.slice(1 + 13);
+ expect(afterTail.slice(0, 6)).toEqual([1000, 2000, 4000, 5000, 5000, 5000]);
+ });
+
+ it("keeps today's schedule when there is no page estimate", async () => {
+ const { sleeps } = await runPoll({ doneAfterPolls: 7 });
+ expect(sleeps).toEqual([1000, 2000, 4000, 5000, 5000, 5000, 5000]);
+ });
+
+ it("keeps today's schedule for a document larger than the bucket table", async () => {
+ const { sleeps } = await runPoll({
+ pagesEstimate: 400,
+ doneAfterPolls: 7,
+ });
+ expect(sleeps).toEqual([1000, 2000, 4000, 5000, 5000, 5000, 5000]);
+ });
+
+ it("lets fire-pdf's retry_after_ms win over the early schedule when it is larger", async () => {
+ const { sleeps } = await runPoll({
+ pagesEstimate: 3,
+ initialDelay: 2_000,
+ doneAfterPolls: 3,
+ runningBody: { retry_after_ms: 1_500 },
+ });
+ // First poll waits for the submit response's hint (2s > the 1.2s p50);
+ // each running poll then asks for 1.5s, which beats the 300ms tick.
+ expect(sleeps).toEqual([2000, 1500, 1500]);
+ });
+
+ it("caps a very large retry_after_ms at the regular poll cap", async () => {
+ const { sleeps } = await runPoll({
+ pagesEstimate: 3,
+ initialDelay: 30_000,
+ doneAfterPolls: 2,
+ runningBody: { retry_after_ms: 30_000 },
+ });
+ expect(sleeps).toEqual([5000, 5000]);
+ });
+
+ it("restarts the backoff from the floor at handover, whatever the submit hint was", async () => {
+ const { sleeps } = await runPoll({
+ pagesEstimate: 3,
+ initialDelay: 30_000,
+ doneAfterPolls: 3,
+ });
+ expect(sleeps).toEqual([5000, 1000, 2000]);
+ });
+
+ it("ignores a retry_after_ms smaller than the early schedule", async () => {
+ const { sleeps } = await runPoll({
+ pagesEstimate: 3,
+ initialDelay: 100,
+ doneAfterPolls: 3,
+ runningBody: { retry_after_ms: 50 },
+ });
+ expect(sleeps).toEqual([1200, 300, 300]);
+ });
+
+ it("caps the number of fast polls per job, however long the slow tail", async () => {
+ // 100 pages: p50 11.8s, p90 31s. 30 fast polls only span ~9s of that
+ // window, so the cap β€” not p90 β€” hands over to the backoff.
+ const { sleeps } = await runPoll({
+ pagesEstimate: 100,
+ doneAfterPolls: 40,
+ });
+ expect(sleeps[0]).toBe(11_800);
+ expect(sleeps.slice(1, 31)).toEqual(repeat(300, 30));
+ expect(sleeps.slice(31, 37)).toEqual([1000, 2000, 4000, 5000, 5000, 5000]);
+ });
+
+ it("still pulls polls in to land just after the job deadline, past the slow tail", async () => {
+ // 3 pages with an 8s job deadline: the early schedule ends at 5.1s,
+ // backoff runs 1s (6.1s) and 2s (8.1s); the next 4s sleep would
+ // overshoot deadline + 1s grace (9s), so it is cut to the 1s floor and
+ // polling stays at the floor after the deadline.
+ const { sleeps } = await runPoll({
+ pagesEstimate: 3,
+ jobDeadlineAtMs: 8_000,
+ doneAfterPolls: 20,
+ });
+ expect(sleeps.slice(14, 18)).toEqual([1000, 2000, 1000, 1000]);
+ });
+});
+
+describe("earlyPollDelay", () => {
+ const base = {
+ elapsedMs: 0,
+ pollCount: 0,
+ fastPollCount: 0,
+ retryAfterMs: undefined,
+ random: () => 0,
+ };
+
+ it("applies +0-20% jitter to the ~300ms fast interval", () => {
+ const delays = [0, 0.5, 0.999].map(random =>
+ earlyPollDelay({
+ ...base,
+ pagesEstimate: 10,
+ pollCount: 3,
+ elapsedMs: 4_000,
+ random: () => random,
+ }),
+ );
+ expect(delays).toEqual([300, 330, 360]);
+ });
+
+ it("picks the bucket from the page estimate", () => {
+ const first = (pages: number) =>
+ earlyPollDelay({ ...base, pagesEstimate: pages });
+ expect(first(1)).toBe(1000);
+ expect(first(5)).toBe(1200);
+ expect(first(6)).toBe(2500);
+ expect(first(25)).toBe(4400);
+ expect(first(50)).toBe(5200);
+ expect(first(100)).toBe(11_800);
+ });
+
+ it("declines without a usable estimate or beyond the table", () => {
+ expect(earlyPollDelay({ ...base, pagesEstimate: undefined })).toBe(
+ undefined,
+ );
+ expect(earlyPollDelay({ ...base, pagesEstimate: 0 })).toBe(undefined);
+ expect(earlyPollDelay({ ...base, pagesEstimate: NaN })).toBe(undefined);
+ expect(earlyPollDelay({ ...base, pagesEstimate: 101 })).toBe(undefined);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 87f6f88ec..3b21981ae 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -19,7 +19,7 @@ import { fetchResult } from "./result";
import type { FirePdfByReferenceInput } from "./by-reference";
import type { FirePdfAdoptedJobInput } from "./lookup";
import { FIRE_PDF_ASYNC_MIN_REMAINING_MS } from "./routing";
-import { POLL_FLOOR_MS, POLL_TIMEOUT_BUFFER_MS } from "./schema";
+import { POLL_TIMEOUT_BUFFER_MS } from "./schema";
import { submitJob, SubmitJobMayHaveBeenAcceptedError } from "./submit";
import {
computeByReferenceDeadlineMs,
@@ -219,7 +219,7 @@ export async function scrapePDFWithFirePDFAsync(
try {
let alreadyDone = false;
- let initialDelay: number = POLL_FLOOR_MS;
+ let initialDelay: number | undefined;
if (wireInput === null) {
meta.logger.info("FirePDF async adopting existing job", {
scrapeId: meta.id,
@@ -256,7 +256,7 @@ export async function scrapePDFWithFirePDFAsync(
});
submissionAccepted = true;
alreadyDone = submit.alreadyDone;
- initialDelay = submit.retryAfterMs ?? POLL_FLOOR_MS;
+ initialDelay = submit.retryAfterMs;
if (byReference && !alreadyDone && meta.largePdfProcessing) {
// The job now exists server-side and (per the cancel policy
// above) will keep running if this scrape is abandoned β€” record
@@ -284,6 +284,7 @@ export async function scrapePDFWithFirePDFAsync(
baseUrl,
scrapeId: jobScrapeId,
initialDelay,
+ pagesEstimate: pagesProcessed,
pollingDeadline,
meta,
fetchImpl,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/early-poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/early-poll.ts
new file mode 100644
index 000000000..9750017f5
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/early-poll.ts
@@ -0,0 +1,81 @@
+// Page-count-aware early polling for the async client.
+//
+// Backoff from a 1s floor (1s, 3s, 7s, 12s, ...) lands far from the moment a
+// short document finishes, so its result sits unseen until the next poll. When
+// the caller has a page estimate, the expected pipeline time is known well
+// enough to do better: one poll around the typical finish time, then a short
+// fixed interval through the slow tail, then the regular backoff.
+//
+// Self-contained on purpose: a server-side long-poll makes this schedule
+// unnecessary, and removing it is deleting this file plus its one call in
+// poll.ts.
+
+import { POLL_CAP_MS } from "./schema";
+
+type PipelineBucket = {
+ /** Inclusive upper bound of the page estimate this row applies to. */
+ maxPages: number;
+ /** Typical pipeline time: where the first poll lands. */
+ p50Ms: number;
+ /** Slow-tail pipeline time: where fast polling stops. */
+ p90Ms: number;
+};
+
+const PIPELINE_BUCKETS: readonly PipelineBucket[] = [
+ { maxPages: 1, p50Ms: 1_000, p90Ms: 3_000 },
+ { maxPages: 5, p50Ms: 1_200, p90Ms: 5_000 },
+ { maxPages: 10, p50Ms: 2_500, p90Ms: 9_700 },
+ { maxPages: 25, p50Ms: 4_400, p90Ms: 14_900 },
+ { maxPages: 50, p50Ms: 5_200, p90Ms: 19_500 },
+ { maxPages: 100, p50Ms: 11_800, p90Ms: 31_000 },
+];
+
+const FAST_POLL_MS = 300;
+const JITTER = 0.2;
+/** Hard floor on any early poll, whatever the constants above say. */
+const MIN_EARLY_POLL_MS = 250;
+/** Bounds the extra requests one job can cause, however long its tail. */
+const MAX_FAST_POLLS = 30;
+
+type EarlyPollState = {
+ pagesEstimate: number | undefined;
+ /** Time since polling began. */
+ elapsedMs: number;
+ /** Polls already sent (the first poll is number 0). */
+ pollCount: number;
+ /** Polls already sent on the fast interval. */
+ fastPollCount: number;
+ /** fire-pdf's own requested delay; wins whenever it is larger. */
+ retryAfterMs: number | undefined;
+ random: () => number;
+};
+
+/**
+ * The delay before the next poll while the job is inside its expected
+ * pipeline window, or undefined once the regular backoff should take over
+ * (no page estimate, a document beyond the table, past the slow tail, or
+ * out of fast polls).
+ */
+export function earlyPollDelay(state: EarlyPollState): number | undefined {
+ const { pagesEstimate } = state;
+ if (pagesEstimate === undefined || !(pagesEstimate > 0)) return undefined;
+ const bucket = PIPELINE_BUCKETS.find(b => pagesEstimate <= b.maxPages);
+ if (!bucket) return undefined;
+
+ let base: number;
+ if (state.pollCount === 0) {
+ base = bucket.p50Ms;
+ } else if (
+ state.elapsedMs < bucket.p90Ms &&
+ state.fastPollCount < MAX_FAST_POLLS
+ ) {
+ base = FAST_POLL_MS;
+ } else {
+ return undefined;
+ }
+ const jittered = Math.round(base * (1 + state.random() * JITTER));
+ // A fire-pdf hint is honoured up to the same ceiling as the backoff, so a
+ // large one never stretches a poll beyond what it could before.
+ const hint = Math.min(POLL_CAP_MS, state.retryAfterMs ?? 0);
+ return Math.max(MIN_EARLY_POLL_MS, jittered, hint);
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
index 5d3afdbce..7417b9841 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
@@ -11,6 +11,7 @@ import {
TERMINAL_STATUSES,
type PollResponse,
} from "./schema";
+import { earlyPollDelay } from "./early-poll";
import {
alignPollDelay,
failAsync,
@@ -21,7 +22,11 @@ import {
type PollDeps = {
baseUrl: string;
scrapeId: string;
- initialDelay: number;
+ /** fire-pdf's `retry_after_ms` from the submit response, when it sent one. */
+ initialDelay?: number;
+ /** Page count of the document, when the caller has one. Selects the early
+ * poll schedule (see early-poll.ts); absent, plain backoff applies. */
+ pagesEstimate?: number;
pollingDeadline: number;
meta: Meta;
fetchImpl: typeof undiciFetch;
@@ -50,6 +55,10 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
let pollCount = 0;
const random = deps.random ?? Math.random;
let lastDelay = nextPollDelay(0, deps.initialDelay, random);
+ const startedAt = now();
+ let retryAfterMs = deps.initialDelay;
+ let fastPollCount = 0;
+ let inEarlySchedule = false;
while (true) {
if (now() > pollingDeadline) {
@@ -58,8 +67,25 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
}
meta.abort.throwIfAborted();
+ const early = earlyPollDelay({
+ pagesEstimate: deps.pagesEstimate,
+ elapsedMs: now() - startedAt,
+ pollCount,
+ fastPollCount,
+ retryAfterMs,
+ random,
+ });
+ if (early !== undefined && pollCount > 0) fastPollCount++;
+ if (early !== undefined) {
+ inEarlySchedule = true;
+ } else if (inEarlySchedule) {
+ // Handover: backoff restarts from the floor (or the latest hint),
+ // not from the seed computed before the early polls began.
+ inEarlySchedule = false;
+ lastDelay = nextPollDelay(0, retryAfterMs, random);
+ }
await sleep(
- alignPollDelay(lastDelay, now(), deps.jobDeadlineAtMs),
+ alignPollDelay(early ?? lastDelay, now(), deps.jobDeadlineAtMs),
meta.abort.asSignal(),
);
pollCount++;
@@ -166,6 +192,10 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
);
}
- lastDelay = nextPollDelay(lastDelay, parsed.data.retry_after_ms, random);
+ retryAfterMs = parsed.data.retry_after_ms;
+ // Backoff advances only while it is the schedule in use.
+ if (early === undefined) {
+ lastDelay = nextPollDelay(lastDelay, retryAfterMs, random);
+ }
}
}