Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): long-poll fire-pdf job status with wait_ms (#4886)
Abimael Martell4279c44f56c9
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 699fb8377..9eeca9fd6 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -433,6 +433,11 @@ const configSchema = z.object({FIRE_PDF_ASYNC_FORCE_TEAM_IDS: z.string().optional(),FIRE_PDF_ASYNC_DISABLE_TEAM_IDS: z.string().optional(),FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE: z.stringbool().default(false),+ // Long-poll wait sent as `wait_ms` on GET /jobs/:id: fire-pdf holds the+ // request until the job is terminal or the wait elapses, so completion is+ // seen as it happens instead of at the next scheduled poll. 0 disables+ // it; values above fire-pdf's 25s cap are clamped to it.+ FIRE_PDF_ASYNC_WAIT_MS: z.coerce.number().int().min(0).default(0),// Large-PDF by-reference submits (30-256MB files uploaded to GCS and// handed to fire-pdf via `input_gcs_uri`). This is an explicit on/off// switch, not a percentage: no alternative engine exists at this size,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFLongPoll.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFLongPoll.test.tsnew file mode 100644index 000000000..899c41f3e--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFLongPoll.test.ts@@ -0,0 +1,264 @@+import { firePdfAsyncLongPollTotal } from "../fire-pdf/metrics";+import { longPollWaitFor, pollUntilTerminal } from "../fire-pdf/poll";+import { counterValue, jsonResp, makeMeta } from "./firePDFAsyncFixtures";++const T0 = 1_000_000;+const DEADLINE = T0 + 10 * 60_000;++// Drives pollUntilTerminal on a virtual clock against a fake fire-pdf.+// `holding` decides whether the fake honours wait_ms: when it does, a+// wait_ms request is answered when the job finishes or the wait elapses,+// whichever comes first; when it doesn't, every request answers at once.+async function runLongPoll(opts: {+ longPollWaitMs?: number;+ doneAtMs: number;+ holding: boolean | ((requestIndex: number) => boolean);+ pollingDeadline?: number;+ pagesEstimate?: number;+}) {+ let virtualNow = T0;+ const sleeps: number[] = [];+ const urls: string[] = [];+ const fetchImpl: any = async (url: string) => {+ urls.push(url);+ const wait = Number(new URL(url).searchParams.get("wait_ms") ?? 0);+ const doneAt = T0 + opts.doneAtMs;+ const holds =+ typeof opts.holding === "function"+ ? opts.holding(urls.length - 1)+ : opts.holding;+ if (holds && wait > 0 && virtualNow < doneAt) {+ virtualNow = Math.min(doneAt, virtualNow + wait);+ }+ return virtualNow >= doneAt+ ? jsonResp({+ status: 200,+ body: { scrape_id: "x", status: "done", pages_processed: 1 },+ })+ : jsonResp({+ status: 202,+ body: { scrape_id: "x", status: "running", retry_after_ms: 250 },+ });+ };+ const result = await pollUntilTerminal({+ baseUrl: "http://fire-pdf.test",+ scrapeId: "x",+ pagesEstimate: opts.pagesEstimate,+ pollingDeadline: opts.pollingDeadline ?? DEADLINE,+ meta: makeMeta(),+ fetchImpl,+ sleep: async ms => {+ sleeps.push(ms);+ virtualNow += ms;+ },+ now: () => virtualNow,+ random: () => 0,+ longPollWaitMs: opts.longPollWaitMs,+ });+ return {+ sleeps,+ urls,+ pollCount: result.pollCount,+ seenAtMs: virtualNow - T0,+ };+}++const waitParam = (url: string) =>+ new URL(url).searchParams.get("wait_ms") ?? undefined;++describe("longPollWaitFor", () => {+ it("uses the configured wait while the deadline is far", () => {+ expect(longPollWaitFor(20_000, 5 * 60_000)).toBe(20_000);+ });++ it("shortens the wait to end before the polling deadline", () => {+ expect(longPollWaitFor(20_000, 6_000)).toBe(5_000);+ });++ it("clamps the wait to fire-pdf's 25s cap", () => {+ expect(longPollWaitFor(60_000, 5 * 60_000)).toBe(25_000);+ });++ it("returns 0 when too little time is left to hold a request", () => {+ expect(longPollWaitFor(20_000, 1_500)).toBe(0);+ expect(longPollWaitFor(0, 5 * 60_000)).toBe(0);+ });+});++describe("pollUntilTerminal — wait_ms long-poll", () => {+ it("keeps today's schedule and URL when the wait is not configured", async () => {+ const { sleeps, urls } = await runLongPoll({+ doneAtMs: 2_500,+ holding: true,+ });+ expect(urls.every(u => waitParam(u) === undefined)).toBe(true);+ expect(sleeps.length).toBeGreaterThan(0);+ });++ it("sends the first poll at once with wait_ms and sees completion when it happens", async () => {+ const before = await counterValue(firePdfAsyncLongPollTotal, {+ outcome: "terminal",+ });+ const { sleeps, urls, pollCount, seenAtMs } = await runLongPoll({+ longPollWaitMs: 20_000,+ doneAtMs: 2_500,+ holding: true,+ });+ expect(sleeps).toEqual([]);+ expect(urls.map(waitParam)).toEqual(["20000"]);+ expect(pollCount).toBe(1);+ expect(seenAtMs).toBe(2_500);+ expect(+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "terminal" }),+ ).toBe(before + 1);+ });++ it("re-polls immediately after a held request times out", async () => {+ const before = await counterValue(firePdfAsyncLongPollTotal, {+ outcome: "held",+ });+ const { sleeps, urls, seenAtMs } = await runLongPoll({+ longPollWaitMs: 20_000,+ doneAtMs: 45_000,+ holding: true,+ });+ expect(sleeps).toEqual([]);+ expect(urls.map(waitParam)).toEqual(["20000", "20000", "20000"]);+ expect(seenAtMs).toBe(45_000);+ expect(+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "held" }),+ ).toBe(before + 2);+ });++ it("falls back to the scheduled polls when the server keeps answering a wait at once", async () => {+ const before = await counterValue(firePdfAsyncLongPollTotal, {+ outcome: "not_held",+ });+ const { sleeps, urls, seenAtMs } = await runLongPoll({+ longPollWaitMs: 20_000,+ doneAtMs: 4_000,+ holding: false,+ pagesEstimate: 3,+ });+ // Two wait_ms requests answered early (a pause between them), then+ // plain polls on the schedule: never a tight loop.+ expect(urls.slice(0, 2).map(waitParam)).toEqual(["20000", "20000"]);+ expect(urls.slice(2).every(u => waitParam(u) === undefined)).toBe(true);+ expect(sleeps[0]).toBe(1_000);+ expect(sleeps.length).toBe(urls.length - 1);+ expect(sleeps.every(ms => ms >= 250)).toBe(true);+ expect(seenAtMs).toBeGreaterThanOrEqual(4_000);+ expect(+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "not_held" }),+ ).toBe(before + 2);+ });++ it("keeps long-polling after a single early answer", async () => {+ const { sleeps, urls, seenAtMs } = await runLongPoll({+ longPollWaitMs: 20_000,+ doneAtMs: 30_000,+ holding: i => i !== 0,+ });+ // Early answer, one floor pause, then held long-polls to completion.+ expect(urls.every(u => waitParam(u) === "20000")).toBe(true);+ expect(sleeps).toEqual([1_000]);+ expect(seenAtMs).toBe(30_000);+ });++ it("sizes the wait after the pause that follows an early answer", async () => {+ const { urls, sleeps } = await runLongPoll({+ longPollWaitMs: 20_000,+ doneAtMs: 5_000,+ holding: i => i !== 0,+ pollingDeadline: T0 + 10_000,+ });+ // 9s left at the first request; after the 1s pause, 8s.+ expect(urls.map(waitParam)).toEqual(["9000", "8000"]);+ expect(sleeps).toEqual([1_000]);+ });++ it("skips the early-answer pause when no long-poll fits after it", async () => {+ const { urls, sleeps, seenAtMs } = await runLongPoll({+ longPollWaitMs: 20_000,+ doneAtMs: 1_000,+ holding: false,+ pollingDeadline: T0 + 2_500,+ });+ // 2.5s left: after the early answer a 1s pause would leave too little+ // for another long-poll, so that round takes the scheduled path (its+ // sleep, then a plain poll), finishing inside the deadline.+ expect(urls.map(waitParam)).toEqual(["1500", undefined]);+ expect(sleeps).toEqual([1_000]);+ expect(seenAtMs).toBeLessThanOrEqual(2_500);+ });++ it("caps a large retry_after_ms hint in the early-answer pause", async () => {+ let virtualNow = T0;+ const sleeps: number[] = [];+ let calls = 0;+ const fetchImpl: any = async () => {+ calls++;+ return calls >= 2+ ? jsonResp({+ status: 200,+ body: { scrape_id: "x", status: "done", pages_processed: 1 },+ })+ : jsonResp({+ status: 202,+ body: { scrape_id: "x", status: "running", retry_after_ms: 30_000 },+ });+ };+ await pollUntilTerminal({+ baseUrl: "http://fire-pdf.test",+ scrapeId: "x",+ pollingDeadline: DEADLINE,+ meta: makeMeta(),+ fetchImpl,+ sleep: async ms => {+ sleeps.push(ms);+ virtualNow += ms;+ },+ now: () => virtualNow,+ random: () => 0,+ longPollWaitMs: 20_000,+ });+ expect(sleeps).toEqual([5_000]);+ });++ it("counts an expired or cancelled answer to a long-poll as terminal", async () => {+ const before = await counterValue(firePdfAsyncLongPollTotal, {+ outcome: "terminal",+ });+ const fetchImpl: any = async () =>+ jsonResp({+ status: 410,+ body: { scrape_id: "x", status: "expired" },+ });+ await expect(+ pollUntilTerminal({+ baseUrl: "http://fire-pdf.test",+ scrapeId: "x",+ pollingDeadline: DEADLINE,+ meta: makeMeta(),+ fetchImpl,+ sleep: async () => {},+ now: () => T0,+ random: () => 0,+ longPollWaitMs: 20_000,+ }),+ ).rejects.toMatchObject({ reason: "terminal_expired" });+ expect(+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "terminal" }),+ ).toBe(before + 1);+ });++ it("bounds the wait by the polling deadline", async () => {+ const { urls } = await runLongPoll({+ longPollWaitMs: 20_000,+ doneAtMs: 3_000,+ holding: true,+ pollingDeadline: T0 + 8_000,+ });+ expect(waitParam(urls[0])).toBe("7000");+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex 3b21981ae..a5120d482 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -285,6 +285,7 @@ export async function scrapePDFWithFirePDFAsync(scrapeId: jobScrapeId,initialDelay,pagesEstimate: pagesProcessed,+ longPollWaitMs: config.FIRE_PDF_ASYNC_WAIT_MS,pollingDeadline,meta,fetchImpl,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex e9b114dbb..97b637564 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -55,6 +55,12 @@ export const firePdfAsyncSubmit503Total = new Counter({labelNames: ["code"],});+export const firePdfAsyncLongPollTotal = new Counter({+ name: "firecrawl_fire_pdf_async_long_poll_total",+ help: "GET /jobs/:id requests sent with wait_ms, by outcome: terminal (answered with a terminal status), held (non-terminal after holding), not_held (non-terminal answered early; followed by a pause, and by scheduled polling after repeated early answers)",+ labelNames: ["outcome"],+});+export const firePdfAsyncTotalDurationSeconds = new Histogram({name: "firecrawl_fire_pdf_async_total_duration_seconds",help: "End-to-end duration from 'decide to use async' to 'result available'",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.tsindex 7417b9841..991ca2290 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts@@ -3,10 +3,18 @@ import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import {firePdfAsyncCompletedTotal,+ firePdfAsyncLongPollTotal,firePdfAsyncPollCount,type FallbackReason,} from "./metrics";import {+ LONG_POLL_DEADLINE_SLACK_MS,+ LONG_POLL_HELD_FRACTION,+ LONG_POLL_MAX_EARLY_ANSWERS,+ LONG_POLL_MAX_WAIT_MS,+ LONG_POLL_MIN_WAIT_MS,+ POLL_CAP_MS,+ POLL_FLOOR_MS,pollResponseSchema,TERMINAL_STATUSES,type PollResponse,@@ -45,8 +53,25 @@ type PollDeps = {status: "queued" | "published" | "running",estimatedRemainingMs?: number,) => void;+ /** Long-poll wait to request as `wait_ms` (FIRE_PDF_ASYNC_WAIT_MS). 0 or+ * absent keeps the scheduled polling below unchanged. */+ longPollWaitMs?: number;};+/** The `wait_ms` to send now: the configured wait, bounded by the polling+ * deadline, or 0 when too little time is left to be worth holding. */+export function longPollWaitFor(+ configuredMs: number,+ msUntilDeadline: number,+): number {+ const wait = Math.min(+ configuredMs,+ LONG_POLL_MAX_WAIT_MS,+ msUntilDeadline - LONG_POLL_DEADLINE_SLACK_MS,+ );+ return wait >= LONG_POLL_MIN_WAIT_MS ? Math.floor(wait) : 0;+}+type PollOk = { poll: PollResponse; pollCount: number };export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {@@ -59,40 +84,77 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {let retryAfterMs = deps.initialDelay;let fastPollCount = 0;let inEarlySchedule = false;+ // Cleared for the rest of this job after LONG_POLL_MAX_EARLY_ANSWERS+ // wait_ms requests in a row come back early without a terminal status.+ let longPollActive = (deps.longPollWaitMs ?? 0) > 0;+ let earlyAnswers = 0;while (true) {+ // After an early answer, pause once on the regular floor before the+ // next wait_ms request. It runs before the deadline check and before+ // the wait is sized, so a held request never outlives the deadline.+ // When no long-poll would fit after it, this round takes the scheduled+ // path instead (its sleep, then a plain poll).+ // The hint is capped like every other use of it in this module.+ let scheduledThisRound = false;+ if (longPollActive && earlyAnswers > 0) {+ const pauseMs = Math.max(+ POLL_FLOOR_MS,+ Math.min(POLL_CAP_MS, retryAfterMs ?? 0),+ );+ const fitsAfterPause =+ longPollWaitFor(+ deps.longPollWaitMs ?? 0,+ pollingDeadline - now() - pauseMs,+ ) > 0;+ if (fitsAfterPause) await sleep(pauseMs, meta.abort.asSignal());+ else scheduledThisRound = true;+ }if (now() > pollingDeadline) {firePdfAsyncPollCount.observe(pollCount);failAsync(meta, "polling_timeout", { pollCount });}meta.abort.throwIfAborted();- const early = earlyPollDelay({- pagesEstimate: deps.pagesEstimate,- elapsedMs: now() - startedAt,- pollCount,- fastPollCount,- retryAfterMs,- random,- });- if (early !== undefined && pollCount > 0) fastPollCount++;- if (early !== undefined) {- inEarlySchedule = true;- } else if (inEarlySchedule) {- // Handover: backoff restarts from the floor (or the latest hint),- // not from the seed computed before the early polls began.- inEarlySchedule = false;- lastDelay = nextPollDelay(0, retryAfterMs, random);+ const waitMs =+ longPollActive && !scheduledThisRound+ ? longPollWaitFor(deps.longPollWaitMs ?? 0, pollingDeadline - now())+ : 0;+ // A long-poll is sent right away: the server does the waiting.+ let early: number | undefined;+ if (waitMs === 0) {+ early = earlyPollDelay({+ pagesEstimate: deps.pagesEstimate,+ elapsedMs: now() - startedAt,+ pollCount,+ fastPollCount,+ retryAfterMs,+ random,+ });+ if (early !== undefined && pollCount > 0) fastPollCount++;+ if (early !== undefined) {+ inEarlySchedule = true;+ } else if (inEarlySchedule) {+ // Handover: backoff restarts from the floor (or the latest hint),+ // not from the seed computed before the early polls began.+ inEarlySchedule = false;+ lastDelay = nextPollDelay(0, retryAfterMs, random);+ }+ await sleep(+ alignPollDelay(early ?? lastDelay, now(), deps.jobDeadlineAtMs),+ meta.abort.asSignal(),+ );}- await sleep(- alignPollDelay(early ?? lastDelay, now(), deps.jobDeadlineAtMs),- meta.abort.asSignal(),- );pollCount++;+ const pollUrl =+ waitMs > 0+ ? `${baseUrl}/jobs/${scrapeId}?wait_ms=${waitMs}`+ : `${baseUrl}/jobs/${scrapeId}`;+ const sentAt = now();let pollResp;try {- pollResp = await fetchImpl(`${baseUrl}/jobs/${scrapeId}`, {+ pollResp = await fetchImpl(pollUrl, {method: "GET",headers: firePdfHeaders(),signal: meta.abort.asSignal(),@@ -122,6 +184,7 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {}if (pollStatus === 410) {+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();firePdfAsyncPollCount.observe(pollCount);const parsed = pollResponseSchema.safeParse(pollBody);const status = parsed.success ? parsed.data.status : "expired";@@ -134,6 +197,7 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {}if (pollStatus === 502) {+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();firePdfAsyncPollCount.observe(pollCount);firePdfAsyncCompletedTotal.labels("failed").inc();failAsync(meta, "terminal_failed", {@@ -162,6 +226,7 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {}if (TERMINAL_STATUSES.has(parsed.data.status)) {+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();firePdfAsyncPollCount.observe(pollCount);firePdfAsyncCompletedTotal.labels(parsed.data.status).inc();if (parsed.data.status !== "done") {@@ -192,9 +257,19 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {);}+ if (waitMs > 0) {+ const held = now() - sentAt >= waitMs * LONG_POLL_HELD_FRACTION;+ firePdfAsyncLongPollTotal.labels(held ? "held" : "not_held").inc();+ // Answered early without finishing: pause before the next wait_ms+ // request, and after repeated early answers treat this server as not+ // holding and use the regular schedule for the rest of the job.+ earlyAnswers = held ? 0 : earlyAnswers + 1;+ if (earlyAnswers >= LONG_POLL_MAX_EARLY_ANSWERS) longPollActive = false;+ }+retryAfterMs = parsed.data.retry_after_ms;// Backoff advances only while it is the schedule in use.- if (early === undefined) {+ if (waitMs === 0 && early === undefined) {lastDelay = nextPollDelay(lastDelay, retryAfterMs, random);}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.tsindex 7ca4c17c5..66ac5ee08 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts@@ -9,6 +9,23 @@ export const MIN_DEADLINE_MS = 5_000;export const MAX_DEADLINE_MS = 30 * 60 * 1_000;export const POLL_FLOOR_MS = 1_000;export const POLL_CAP_MS = 5_000;+// Long-poll (`wait_ms`). A wait shorter than LONG_POLL_MIN_WAIT_MS isn't+// worth holding a connection for, so the last stretch before the polling+// deadline runs on the regular schedule. A non-terminal answer that came+// back in under LONG_POLL_HELD_FRACTION of the requested wait means the+// server did not hold it (older build, or its wake-up path unavailable):+// the job drops back to the regular schedule instead of re-polling at once.+// LONG_POLL_MAX_WAIT_MS mirrors fire-pdf's default cap+// (FIRE_PDF_JOBS_LONG_POLL_MAX_WAIT_MS): fire-pdf clamps longer waits, so+// asking for more would only make a held answer look early. After+// LONG_POLL_MAX_EARLY_ANSWERS early answers in a row the job stays on the+// regular schedule; a single early answer is followed by one scheduled+// pause, never an immediate re-poll.+export const LONG_POLL_MIN_WAIT_MS = 1_000;+export const LONG_POLL_MAX_WAIT_MS = 25_000;+export const LONG_POLL_DEADLINE_SLACK_MS = 1_000;+export const LONG_POLL_HELD_FRACTION = 0.5;+export const LONG_POLL_MAX_EARLY_ANSWERS = 2;export const POLL_TIMEOUT_BUFFER_MS = 30_000;// An inline job's deadline sits this far inside the caller's window. The