Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
perf(api): poll fire-pdf jobs on a page-aware early schedule (#4880)
Abimael Martell463213eb8fe6
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.tsindex e29f8dfdf..4dbe60fcb 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts@@ -158,6 +158,60 @@ describe("scrapePDFWithFirePDFAsync β deadline and submit lifecycle", () => {expect(pollTimes).toEqual([1000, 3000, 7000, 12000, 17000, 21000, 22000]);});+ it("polls on the page-aware early schedule when the caller passes a page estimate", async () => {+ let virtualNow = 1_000_000;+ const sleeps: number[] = [];+ let polls = 0;+ const fetchImpl: any = async (url: string, init: any) => {+ const method = (init?.method ?? "GET").toUpperCase();+ if (/\/jobs$/.test(url) && method === "POST") {+ return jsonResp({+ status: 202,+ body: { scrape_id: "x", status: "queued", lane: "standard" },+ });+ }+ if (/\/jobs\/scrape-id-test$/.test(url)) {+ polls++;+ return jsonResp(+ polls <= 3+ ? { status: 202, body: { scrape_id: "x", status: "running" } }+ : {+ status: 200,+ body: { scrape_id: "x", status: "done", pages_processed: 3 },+ },+ );+ }+ return jsonResp({+ status: 200,+ body: { markdown: "ok", pages_processed: 3 },+ });+ };++ // Same flow as above but with a 3-page estimate (inline and+ // by-reference callers both pass it): first poll at the 3-page p50+ // rather than the 1s floor, then the fast interval.+ const result = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ 3,+ undefined,+ {+ fetchImpl,+ fallbackImpl: vi.fn(),+ sleepImpl: async ms => {+ sleeps.push(ms);+ virtualNow += ms;+ },+ nowImpl: () => virtualNow,+ randomImpl: () => 0,+ },+ );++ expect(result.markdown).toBe("ok");+ expect(sleeps).toEqual([1200, 300, 300, 300]);+ });+it("retries the submit once when a closing api pod answers with Fastify's 503 body", async () => {const before = await counterValue(firePdfAsyncSubmitRetriesTotal, {trigger: "http_503_closing",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFEarlyPoll.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFEarlyPoll.test.tsnew file mode 100644index 000000000..e66bc78f2--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFEarlyPoll.test.ts@@ -0,0 +1,215 @@+import { earlyPollDelay } from "../fire-pdf/early-poll";+import { pollUntilTerminal } from "../fire-pdf/poll";+import { jsonResp, makeMeta } from "./firePDFAsyncFixtures";++const T0 = 1_000_000;++// Drives pollUntilTerminal on a virtual clock. The job is `running` until+// `doneAtMs` of virtual time has passed (or `doneAfterPolls` polls were+// served), and `runningBody` is what every running poll answers with.+async function runPoll(opts: {+ pagesEstimate?: number;+ initialDelay?: number;+ doneAtMs?: number;+ doneAfterPolls?: number;+ jobDeadlineAtMs?: number;+ runningBody?: Record<string, unknown>;+}) {+ let virtualNow = T0;+ const sleeps: number[] = [];+ let polls = 0;+ const fetchImpl: any = async () => {+ polls++;+ const done =+ (opts.doneAtMs !== undefined && virtualNow - T0 >= opts.doneAtMs) ||+ (opts.doneAfterPolls !== undefined && polls >= opts.doneAfterPolls);+ return done+ ? jsonResp({+ status: 200,+ body: { scrape_id: "x", status: "done", pages_processed: 1 },+ })+ : jsonResp({+ status: 202,+ body: {+ scrape_id: "x",+ status: "running",+ ...opts.runningBody,+ },+ });+ };+ const result = await pollUntilTerminal({+ baseUrl: "http://fire-pdf.test",+ scrapeId: "x",+ initialDelay: opts.initialDelay,+ pagesEstimate: opts.pagesEstimate,+ pollingDeadline: T0 + 10 * 60_000,+ meta: makeMeta(),+ fetchImpl,+ sleep: async ms => {+ sleeps.push(ms);+ virtualNow += ms;+ },+ now: () => virtualNow,+ random: () => 0,+ jobDeadlineAtMs:+ opts.jobDeadlineAtMs !== undefined+ ? T0 + opts.jobDeadlineAtMs+ : undefined,+ });+ return { sleeps, pollCount: result.pollCount };+}++const repeat = (ms: number, n: number) => Array<number>(n).fill(ms);++describe("pollUntilTerminal β page-aware early polling", () => {+ it("polls a small document at its expected finish time, then every ~300ms through the slow tail", async () => {+ // 3 pages: first poll at 1.2s, fast polls while under 5s (the 13th+ // lands at 5.1s), then the regular backoff from its 1s floor.+ const { sleeps, pollCount } = await runPoll({+ pagesEstimate: 3,+ doneAtMs: 6_000,+ });+ expect(sleeps).toEqual([1200, ...repeat(300, 13), 1000]);+ expect(pollCount).toBe(15);+ });++ it("sees a job that finishes at the expected time on the very first poll", async () => {+ const { sleeps, pollCount } = await runPoll({+ pagesEstimate: 1,+ doneAtMs: 1_000,+ });+ expect(sleeps).toEqual([1000]);+ expect(pollCount).toBe(1);+ });++ it("falls back to the doubling backoff, capped at 5s, once past the slow tail", async () => {+ const { sleeps } = await runPoll({+ pagesEstimate: 3,+ doneAtMs: 30_000,+ });+ const afterTail = sleeps.slice(1 + 13);+ expect(afterTail.slice(0, 6)).toEqual([1000, 2000, 4000, 5000, 5000, 5000]);+ });++ it("keeps today's schedule when there is no page estimate", async () => {+ const { sleeps } = await runPoll({ doneAfterPolls: 7 });+ expect(sleeps).toEqual([1000, 2000, 4000, 5000, 5000, 5000, 5000]);+ });++ it("keeps today's schedule for a document larger than the bucket table", async () => {+ const { sleeps } = await runPoll({+ pagesEstimate: 400,+ doneAfterPolls: 7,+ });+ expect(sleeps).toEqual([1000, 2000, 4000, 5000, 5000, 5000, 5000]);+ });++ it("lets fire-pdf's retry_after_ms win over the early schedule when it is larger", async () => {+ const { sleeps } = await runPoll({+ pagesEstimate: 3,+ initialDelay: 2_000,+ doneAfterPolls: 3,+ runningBody: { retry_after_ms: 1_500 },+ });+ // First poll waits for the submit response's hint (2s > the 1.2s p50);+ // each running poll then asks for 1.5s, which beats the 300ms tick.+ expect(sleeps).toEqual([2000, 1500, 1500]);+ });++ it("caps a very large retry_after_ms at the regular poll cap", async () => {+ const { sleeps } = await runPoll({+ pagesEstimate: 3,+ initialDelay: 30_000,+ doneAfterPolls: 2,+ runningBody: { retry_after_ms: 30_000 },+ });+ expect(sleeps).toEqual([5000, 5000]);+ });++ it("restarts the backoff from the floor at handover, whatever the submit hint was", async () => {+ const { sleeps } = await runPoll({+ pagesEstimate: 3,+ initialDelay: 30_000,+ doneAfterPolls: 3,+ });+ expect(sleeps).toEqual([5000, 1000, 2000]);+ });++ it("ignores a retry_after_ms smaller than the early schedule", async () => {+ const { sleeps } = await runPoll({+ pagesEstimate: 3,+ initialDelay: 100,+ doneAfterPolls: 3,+ runningBody: { retry_after_ms: 50 },+ });+ expect(sleeps).toEqual([1200, 300, 300]);+ });++ it("caps the number of fast polls per job, however long the slow tail", async () => {+ // 100 pages: p50 11.8s, p90 31s. 30 fast polls only span ~9s of that+ // window, so the cap β not p90 β hands over to the backoff.+ const { sleeps } = await runPoll({+ pagesEstimate: 100,+ doneAfterPolls: 40,+ });+ expect(sleeps[0]).toBe(11_800);+ expect(sleeps.slice(1, 31)).toEqual(repeat(300, 30));+ expect(sleeps.slice(31, 37)).toEqual([1000, 2000, 4000, 5000, 5000, 5000]);+ });++ it("still pulls polls in to land just after the job deadline, past the slow tail", async () => {+ // 3 pages with an 8s job deadline: the early schedule ends at 5.1s,+ // backoff runs 1s (6.1s) and 2s (8.1s); the next 4s sleep would+ // overshoot deadline + 1s grace (9s), so it is cut to the 1s floor and+ // polling stays at the floor after the deadline.+ const { sleeps } = await runPoll({+ pagesEstimate: 3,+ jobDeadlineAtMs: 8_000,+ doneAfterPolls: 20,+ });+ expect(sleeps.slice(14, 18)).toEqual([1000, 2000, 1000, 1000]);+ });+});++describe("earlyPollDelay", () => {+ const base = {+ elapsedMs: 0,+ pollCount: 0,+ fastPollCount: 0,+ retryAfterMs: undefined,+ random: () => 0,+ };++ it("applies +0-20% jitter to the ~300ms fast interval", () => {+ const delays = [0, 0.5, 0.999].map(random =>+ earlyPollDelay({+ ...base,+ pagesEstimate: 10,+ pollCount: 3,+ elapsedMs: 4_000,+ random: () => random,+ }),+ );+ expect(delays).toEqual([300, 330, 360]);+ });++ it("picks the bucket from the page estimate", () => {+ const first = (pages: number) =>+ earlyPollDelay({ ...base, pagesEstimate: pages });+ expect(first(1)).toBe(1000);+ expect(first(5)).toBe(1200);+ expect(first(6)).toBe(2500);+ expect(first(25)).toBe(4400);+ expect(first(50)).toBe(5200);+ expect(first(100)).toBe(11_800);+ });++ it("declines without a usable estimate or beyond the table", () => {+ expect(earlyPollDelay({ ...base, pagesEstimate: undefined })).toBe(+ undefined,+ );+ expect(earlyPollDelay({ ...base, pagesEstimate: 0 })).toBe(undefined);+ expect(earlyPollDelay({ ...base, pagesEstimate: NaN })).toBe(undefined);+ expect(earlyPollDelay({ ...base, pagesEstimate: 101 })).toBe(undefined);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 87f6f88ec..3b21981ae 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -19,7 +19,7 @@ import { fetchResult } from "./result";import type { FirePdfByReferenceInput } from "./by-reference";import type { FirePdfAdoptedJobInput } from "./lookup";import { FIRE_PDF_ASYNC_MIN_REMAINING_MS } from "./routing";-import { POLL_FLOOR_MS, POLL_TIMEOUT_BUFFER_MS } from "./schema";+import { POLL_TIMEOUT_BUFFER_MS } from "./schema";import { submitJob, SubmitJobMayHaveBeenAcceptedError } from "./submit";import {computeByReferenceDeadlineMs,@@ -219,7 +219,7 @@ export async function scrapePDFWithFirePDFAsync(try {let alreadyDone = false;- let initialDelay: number = POLL_FLOOR_MS;+ let initialDelay: number | undefined;if (wireInput === null) {meta.logger.info("FirePDF async adopting existing job", {scrapeId: meta.id,@@ -256,7 +256,7 @@ export async function scrapePDFWithFirePDFAsync(});submissionAccepted = true;alreadyDone = submit.alreadyDone;- initialDelay = submit.retryAfterMs ?? POLL_FLOOR_MS;+ initialDelay = submit.retryAfterMs;if (byReference && !alreadyDone && meta.largePdfProcessing) {// The job now exists server-side and (per the cancel policy// above) will keep running if this scrape is abandoned β record@@ -284,6 +284,7 @@ export async function scrapePDFWithFirePDFAsync(baseUrl,scrapeId: jobScrapeId,initialDelay,+ pagesEstimate: pagesProcessed,pollingDeadline,meta,fetchImpl,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/early-poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/early-poll.tsnew file mode 100644index 000000000..9750017f5--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/early-poll.ts@@ -0,0 +1,81 @@+// Page-count-aware early polling for the async client.+//+// Backoff from a 1s floor (1s, 3s, 7s, 12s, ...) lands far from the moment a+// short document finishes, so its result sits unseen until the next poll. When+// the caller has a page estimate, the expected pipeline time is known well+// enough to do better: one poll around the typical finish time, then a short+// fixed interval through the slow tail, then the regular backoff.+//+// Self-contained on purpose: a server-side long-poll makes this schedule+// unnecessary, and removing it is deleting this file plus its one call in+// poll.ts.++import { POLL_CAP_MS } from "./schema";++type PipelineBucket = {+ /** Inclusive upper bound of the page estimate this row applies to. */+ maxPages: number;+ /** Typical pipeline time: where the first poll lands. */+ p50Ms: number;+ /** Slow-tail pipeline time: where fast polling stops. */+ p90Ms: number;+};++const PIPELINE_BUCKETS: readonly PipelineBucket[] = [+ { maxPages: 1, p50Ms: 1_000, p90Ms: 3_000 },+ { maxPages: 5, p50Ms: 1_200, p90Ms: 5_000 },+ { maxPages: 10, p50Ms: 2_500, p90Ms: 9_700 },+ { maxPages: 25, p50Ms: 4_400, p90Ms: 14_900 },+ { maxPages: 50, p50Ms: 5_200, p90Ms: 19_500 },+ { maxPages: 100, p50Ms: 11_800, p90Ms: 31_000 },+];++const FAST_POLL_MS = 300;+const JITTER = 0.2;+/** Hard floor on any early poll, whatever the constants above say. */+const MIN_EARLY_POLL_MS = 250;+/** Bounds the extra requests one job can cause, however long its tail. */+const MAX_FAST_POLLS = 30;++type EarlyPollState = {+ pagesEstimate: number | undefined;+ /** Time since polling began. */+ elapsedMs: number;+ /** Polls already sent (the first poll is number 0). */+ pollCount: number;+ /** Polls already sent on the fast interval. */+ fastPollCount: number;+ /** fire-pdf's own requested delay; wins whenever it is larger. */+ retryAfterMs: number | undefined;+ random: () => number;+};++/**+ * The delay before the next poll while the job is inside its expected+ * pipeline window, or undefined once the regular backoff should take over+ * (no page estimate, a document beyond the table, past the slow tail, or+ * out of fast polls).+ */+export function earlyPollDelay(state: EarlyPollState): number | undefined {+ const { pagesEstimate } = state;+ if (pagesEstimate === undefined || !(pagesEstimate > 0)) return undefined;+ const bucket = PIPELINE_BUCKETS.find(b => pagesEstimate <= b.maxPages);+ if (!bucket) return undefined;++ let base: number;+ if (state.pollCount === 0) {+ base = bucket.p50Ms;+ } else if (+ state.elapsedMs < bucket.p90Ms &&+ state.fastPollCount < MAX_FAST_POLLS+ ) {+ base = FAST_POLL_MS;+ } else {+ return undefined;+ }+ const jittered = Math.round(base * (1 + state.random() * JITTER));+ // A fire-pdf hint is honoured up to the same ceiling as the backoff, so a+ // large one never stretches a poll beyond what it could before.+ const hint = Math.min(POLL_CAP_MS, state.retryAfterMs ?? 0);+ return Math.max(MIN_EARLY_POLL_MS, jittered, hint);+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.tsindex 5d3afdbce..7417b9841 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts@@ -11,6 +11,7 @@ import {TERMINAL_STATUSES,type PollResponse,} from "./schema";+import { earlyPollDelay } from "./early-poll";import {alignPollDelay,failAsync,@@ -21,7 +22,11 @@ import {type PollDeps = {baseUrl: string;scrapeId: string;- initialDelay: number;+ /** fire-pdf's `retry_after_ms` from the submit response, when it sent one. */+ initialDelay?: number;+ /** Page count of the document, when the caller has one. Selects the early+ * poll schedule (see early-poll.ts); absent, plain backoff applies. */+ pagesEstimate?: number;pollingDeadline: number;meta: Meta;fetchImpl: typeof undiciFetch;@@ -50,6 +55,10 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {let pollCount = 0;const random = deps.random ?? Math.random;let lastDelay = nextPollDelay(0, deps.initialDelay, random);+ const startedAt = now();+ let retryAfterMs = deps.initialDelay;+ let fastPollCount = 0;+ let inEarlySchedule = false;while (true) {if (now() > pollingDeadline) {@@ -58,8 +67,25 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {}meta.abort.throwIfAborted();+ const early = earlyPollDelay({+ pagesEstimate: deps.pagesEstimate,+ elapsedMs: now() - startedAt,+ pollCount,+ fastPollCount,+ retryAfterMs,+ random,+ });+ if (early !== undefined && pollCount > 0) fastPollCount++;+ if (early !== undefined) {+ inEarlySchedule = true;+ } else if (inEarlySchedule) {+ // Handover: backoff restarts from the floor (or the latest hint),+ // not from the seed computed before the early polls began.+ inEarlySchedule = false;+ lastDelay = nextPollDelay(0, retryAfterMs, random);+ }await sleep(- alignPollDelay(lastDelay, now(), deps.jobDeadlineAtMs),+ alignPollDelay(early ?? lastDelay, now(), deps.jobDeadlineAtMs),meta.abort.asSignal(),);pollCount++;@@ -166,6 +192,10 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {);}- lastDelay = nextPollDelay(lastDelay, parsed.data.retry_after_ms, random);+ retryAfterMs = parsed.data.retry_after_ms;+ // Backoff advances only while it is the schedule in use.+ if (early === undefined) {+ lastDelay = nextPollDelay(lastDelay, retryAfterMs, random);+ }}}