Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): long-poll fire-pdf job status with wait_ms (#4886)

Abimael Martell4279c44f56c9
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 699fb8377..9eeca9fd6 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -433,6 +433,11 @@ const configSchema = z.object({
FIRE_PDF_ASYNC_FORCE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_DISABLE_TEAM_IDS: z.string().optional(),
FIRE_PDF_ASYNC_ALLOW_REQUEST_OVERRIDE: z.stringbool().default(false),
+ // Long-poll wait sent as `wait_ms` on GET /jobs/:id: fire-pdf holds the
+ // request until the job is terminal or the wait elapses, so completion is
+ // seen as it happens instead of at the next scheduled poll. 0 disables
+ // it; values above fire-pdf's 25s cap are clamped to it.
+ FIRE_PDF_ASYNC_WAIT_MS: z.coerce.number().int().min(0).default(0),
// Large-PDF by-reference submits (30-256MB files uploaded to GCS and
// handed to fire-pdf via `input_gcs_uri`). This is an explicit on/off
// switch, not a percentage: no alternative engine exists at this size,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFLongPoll.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFLongPoll.test.ts
new file mode 100644
index 000000000..899c41f3e
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFLongPoll.test.ts
@@ -0,0 +1,264 @@
+import { firePdfAsyncLongPollTotal } from "../fire-pdf/metrics";
+import { longPollWaitFor, pollUntilTerminal } from "../fire-pdf/poll";
+import { counterValue, jsonResp, makeMeta } from "./firePDFAsyncFixtures";
+
+const T0 = 1_000_000;
+const DEADLINE = T0 + 10 * 60_000;
+
+// Drives pollUntilTerminal on a virtual clock against a fake fire-pdf.
+// `holding` decides whether the fake honours wait_ms: when it does, a
+// wait_ms request is answered when the job finishes or the wait elapses,
+// whichever comes first; when it doesn't, every request answers at once.
+async function runLongPoll(opts: {
+ longPollWaitMs?: number;
+ doneAtMs: number;
+ holding: boolean | ((requestIndex: number) => boolean);
+ pollingDeadline?: number;
+ pagesEstimate?: number;
+}) {
+ let virtualNow = T0;
+ const sleeps: number[] = [];
+ const urls: string[] = [];
+ const fetchImpl: any = async (url: string) => {
+ urls.push(url);
+ const wait = Number(new URL(url).searchParams.get("wait_ms") ?? 0);
+ const doneAt = T0 + opts.doneAtMs;
+ const holds =
+ typeof opts.holding === "function"
+ ? opts.holding(urls.length - 1)
+ : opts.holding;
+ if (holds && wait > 0 && virtualNow < doneAt) {
+ virtualNow = Math.min(doneAt, virtualNow + wait);
+ }
+ return virtualNow >= doneAt
+ ? jsonResp({
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 1 },
+ })
+ : jsonResp({
+ status: 202,
+ body: { scrape_id: "x", status: "running", retry_after_ms: 250 },
+ });
+ };
+ const result = await pollUntilTerminal({
+ baseUrl: "http://fire-pdf.test",
+ scrapeId: "x",
+ pagesEstimate: opts.pagesEstimate,
+ pollingDeadline: opts.pollingDeadline ?? DEADLINE,
+ meta: makeMeta(),
+ fetchImpl,
+ sleep: async ms => {
+ sleeps.push(ms);
+ virtualNow += ms;
+ },
+ now: () => virtualNow,
+ random: () => 0,
+ longPollWaitMs: opts.longPollWaitMs,
+ });
+ return {
+ sleeps,
+ urls,
+ pollCount: result.pollCount,
+ seenAtMs: virtualNow - T0,
+ };
+}
+
+const waitParam = (url: string) =>
+ new URL(url).searchParams.get("wait_ms") ?? undefined;
+
+describe("longPollWaitFor", () => {
+ it("uses the configured wait while the deadline is far", () => {
+ expect(longPollWaitFor(20_000, 5 * 60_000)).toBe(20_000);
+ });
+
+ it("shortens the wait to end before the polling deadline", () => {
+ expect(longPollWaitFor(20_000, 6_000)).toBe(5_000);
+ });
+
+ it("clamps the wait to fire-pdf's 25s cap", () => {
+ expect(longPollWaitFor(60_000, 5 * 60_000)).toBe(25_000);
+ });
+
+ it("returns 0 when too little time is left to hold a request", () => {
+ expect(longPollWaitFor(20_000, 1_500)).toBe(0);
+ expect(longPollWaitFor(0, 5 * 60_000)).toBe(0);
+ });
+});
+
+describe("pollUntilTerminal — wait_ms long-poll", () => {
+ it("keeps today's schedule and URL when the wait is not configured", async () => {
+ const { sleeps, urls } = await runLongPoll({
+ doneAtMs: 2_500,
+ holding: true,
+ });
+ expect(urls.every(u => waitParam(u) === undefined)).toBe(true);
+ expect(sleeps.length).toBeGreaterThan(0);
+ });
+
+ it("sends the first poll at once with wait_ms and sees completion when it happens", async () => {
+ const before = await counterValue(firePdfAsyncLongPollTotal, {
+ outcome: "terminal",
+ });
+ const { sleeps, urls, pollCount, seenAtMs } = await runLongPoll({
+ longPollWaitMs: 20_000,
+ doneAtMs: 2_500,
+ holding: true,
+ });
+ expect(sleeps).toEqual([]);
+ expect(urls.map(waitParam)).toEqual(["20000"]);
+ expect(pollCount).toBe(1);
+ expect(seenAtMs).toBe(2_500);
+ expect(
+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "terminal" }),
+ ).toBe(before + 1);
+ });
+
+ it("re-polls immediately after a held request times out", async () => {
+ const before = await counterValue(firePdfAsyncLongPollTotal, {
+ outcome: "held",
+ });
+ const { sleeps, urls, seenAtMs } = await runLongPoll({
+ longPollWaitMs: 20_000,
+ doneAtMs: 45_000,
+ holding: true,
+ });
+ expect(sleeps).toEqual([]);
+ expect(urls.map(waitParam)).toEqual(["20000", "20000", "20000"]);
+ expect(seenAtMs).toBe(45_000);
+ expect(
+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "held" }),
+ ).toBe(before + 2);
+ });
+
+ it("falls back to the scheduled polls when the server keeps answering a wait at once", async () => {
+ const before = await counterValue(firePdfAsyncLongPollTotal, {
+ outcome: "not_held",
+ });
+ const { sleeps, urls, seenAtMs } = await runLongPoll({
+ longPollWaitMs: 20_000,
+ doneAtMs: 4_000,
+ holding: false,
+ pagesEstimate: 3,
+ });
+ // Two wait_ms requests answered early (a pause between them), then
+ // plain polls on the schedule: never a tight loop.
+ expect(urls.slice(0, 2).map(waitParam)).toEqual(["20000", "20000"]);
+ expect(urls.slice(2).every(u => waitParam(u) === undefined)).toBe(true);
+ expect(sleeps[0]).toBe(1_000);
+ expect(sleeps.length).toBe(urls.length - 1);
+ expect(sleeps.every(ms => ms >= 250)).toBe(true);
+ expect(seenAtMs).toBeGreaterThanOrEqual(4_000);
+ expect(
+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "not_held" }),
+ ).toBe(before + 2);
+ });
+
+ it("keeps long-polling after a single early answer", async () => {
+ const { sleeps, urls, seenAtMs } = await runLongPoll({
+ longPollWaitMs: 20_000,
+ doneAtMs: 30_000,
+ holding: i => i !== 0,
+ });
+ // Early answer, one floor pause, then held long-polls to completion.
+ expect(urls.every(u => waitParam(u) === "20000")).toBe(true);
+ expect(sleeps).toEqual([1_000]);
+ expect(seenAtMs).toBe(30_000);
+ });
+
+ it("sizes the wait after the pause that follows an early answer", async () => {
+ const { urls, sleeps } = await runLongPoll({
+ longPollWaitMs: 20_000,
+ doneAtMs: 5_000,
+ holding: i => i !== 0,
+ pollingDeadline: T0 + 10_000,
+ });
+ // 9s left at the first request; after the 1s pause, 8s.
+ expect(urls.map(waitParam)).toEqual(["9000", "8000"]);
+ expect(sleeps).toEqual([1_000]);
+ });
+
+ it("skips the early-answer pause when no long-poll fits after it", async () => {
+ const { urls, sleeps, seenAtMs } = await runLongPoll({
+ longPollWaitMs: 20_000,
+ doneAtMs: 1_000,
+ holding: false,
+ pollingDeadline: T0 + 2_500,
+ });
+ // 2.5s left: after the early answer a 1s pause would leave too little
+ // for another long-poll, so that round takes the scheduled path (its
+ // sleep, then a plain poll), finishing inside the deadline.
+ expect(urls.map(waitParam)).toEqual(["1500", undefined]);
+ expect(sleeps).toEqual([1_000]);
+ expect(seenAtMs).toBeLessThanOrEqual(2_500);
+ });
+
+ it("caps a large retry_after_ms hint in the early-answer pause", async () => {
+ let virtualNow = T0;
+ const sleeps: number[] = [];
+ let calls = 0;
+ const fetchImpl: any = async () => {
+ calls++;
+ return calls >= 2
+ ? jsonResp({
+ status: 200,
+ body: { scrape_id: "x", status: "done", pages_processed: 1 },
+ })
+ : jsonResp({
+ status: 202,
+ body: { scrape_id: "x", status: "running", retry_after_ms: 30_000 },
+ });
+ };
+ await pollUntilTerminal({
+ baseUrl: "http://fire-pdf.test",
+ scrapeId: "x",
+ pollingDeadline: DEADLINE,
+ meta: makeMeta(),
+ fetchImpl,
+ sleep: async ms => {
+ sleeps.push(ms);
+ virtualNow += ms;
+ },
+ now: () => virtualNow,
+ random: () => 0,
+ longPollWaitMs: 20_000,
+ });
+ expect(sleeps).toEqual([5_000]);
+ });
+
+ it("counts an expired or cancelled answer to a long-poll as terminal", async () => {
+ const before = await counterValue(firePdfAsyncLongPollTotal, {
+ outcome: "terminal",
+ });
+ const fetchImpl: any = async () =>
+ jsonResp({
+ status: 410,
+ body: { scrape_id: "x", status: "expired" },
+ });
+ await expect(
+ pollUntilTerminal({
+ baseUrl: "http://fire-pdf.test",
+ scrapeId: "x",
+ pollingDeadline: DEADLINE,
+ meta: makeMeta(),
+ fetchImpl,
+ sleep: async () => {},
+ now: () => T0,
+ random: () => 0,
+ longPollWaitMs: 20_000,
+ }),
+ ).rejects.toMatchObject({ reason: "terminal_expired" });
+ expect(
+ await counterValue(firePdfAsyncLongPollTotal, { outcome: "terminal" }),
+ ).toBe(before + 1);
+ });
+
+ it("bounds the wait by the polling deadline", async () => {
+ const { urls } = await runLongPoll({
+ longPollWaitMs: 20_000,
+ doneAtMs: 3_000,
+ holding: true,
+ pollingDeadline: T0 + 8_000,
+ });
+ expect(waitParam(urls[0])).toBe("7000");
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index 3b21981ae..a5120d482 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -285,6 +285,7 @@ export async function scrapePDFWithFirePDFAsync(
scrapeId: jobScrapeId,
initialDelay,
pagesEstimate: pagesProcessed,
+ longPollWaitMs: config.FIRE_PDF_ASYNC_WAIT_MS,
pollingDeadline,
meta,
fetchImpl,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
index e9b114dbb..97b637564 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -55,6 +55,12 @@ export const firePdfAsyncSubmit503Total = new Counter({
labelNames: ["code"],
});
+export const firePdfAsyncLongPollTotal = new Counter({
+ name: "firecrawl_fire_pdf_async_long_poll_total",
+ help: "GET /jobs/:id requests sent with wait_ms, by outcome: terminal (answered with a terminal status), held (non-terminal after holding), not_held (non-terminal answered early; followed by a pause, and by scheduled polling after repeated early answers)",
+ labelNames: ["outcome"],
+});
+
export const firePdfAsyncTotalDurationSeconds = new Histogram({
name: "firecrawl_fire_pdf_async_total_duration_seconds",
help: "End-to-end duration from 'decide to use async' to 'result available'",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
index 7417b9841..991ca2290 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
@@ -3,10 +3,18 @@ import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import {
firePdfAsyncCompletedTotal,
+ firePdfAsyncLongPollTotal,
firePdfAsyncPollCount,
type FallbackReason,
} from "./metrics";
import {
+ LONG_POLL_DEADLINE_SLACK_MS,
+ LONG_POLL_HELD_FRACTION,
+ LONG_POLL_MAX_EARLY_ANSWERS,
+ LONG_POLL_MAX_WAIT_MS,
+ LONG_POLL_MIN_WAIT_MS,
+ POLL_CAP_MS,
+ POLL_FLOOR_MS,
pollResponseSchema,
TERMINAL_STATUSES,
type PollResponse,
@@ -45,8 +53,25 @@ type PollDeps = {
status: "queued" | "published" | "running",
estimatedRemainingMs?: number,
) => void;
+ /** Long-poll wait to request as `wait_ms` (FIRE_PDF_ASYNC_WAIT_MS). 0 or
+ * absent keeps the scheduled polling below unchanged. */
+ longPollWaitMs?: number;
};
+/** The `wait_ms` to send now: the configured wait, bounded by the polling
+ * deadline, or 0 when too little time is left to be worth holding. */
+export function longPollWaitFor(
+ configuredMs: number,
+ msUntilDeadline: number,
+): number {
+ const wait = Math.min(
+ configuredMs,
+ LONG_POLL_MAX_WAIT_MS,
+ msUntilDeadline - LONG_POLL_DEADLINE_SLACK_MS,
+ );
+ return wait >= LONG_POLL_MIN_WAIT_MS ? Math.floor(wait) : 0;
+}
+
type PollOk = { poll: PollResponse; pollCount: number };
export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
@@ -59,40 +84,77 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
let retryAfterMs = deps.initialDelay;
let fastPollCount = 0;
let inEarlySchedule = false;
+ // Cleared for the rest of this job after LONG_POLL_MAX_EARLY_ANSWERS
+ // wait_ms requests in a row come back early without a terminal status.
+ let longPollActive = (deps.longPollWaitMs ?? 0) > 0;
+ let earlyAnswers = 0;
while (true) {
+ // After an early answer, pause once on the regular floor before the
+ // next wait_ms request. It runs before the deadline check and before
+ // the wait is sized, so a held request never outlives the deadline.
+ // When no long-poll would fit after it, this round takes the scheduled
+ // path instead (its sleep, then a plain poll).
+ // The hint is capped like every other use of it in this module.
+ let scheduledThisRound = false;
+ if (longPollActive && earlyAnswers > 0) {
+ const pauseMs = Math.max(
+ POLL_FLOOR_MS,
+ Math.min(POLL_CAP_MS, retryAfterMs ?? 0),
+ );
+ const fitsAfterPause =
+ longPollWaitFor(
+ deps.longPollWaitMs ?? 0,
+ pollingDeadline - now() - pauseMs,
+ ) > 0;
+ if (fitsAfterPause) await sleep(pauseMs, meta.abort.asSignal());
+ else scheduledThisRound = true;
+ }
if (now() > pollingDeadline) {
firePdfAsyncPollCount.observe(pollCount);
failAsync(meta, "polling_timeout", { pollCount });
}
meta.abort.throwIfAborted();
- const early = earlyPollDelay({
- pagesEstimate: deps.pagesEstimate,
- elapsedMs: now() - startedAt,
- pollCount,
- fastPollCount,
- retryAfterMs,
- random,
- });
- if (early !== undefined && pollCount > 0) fastPollCount++;
- if (early !== undefined) {
- inEarlySchedule = true;
- } else if (inEarlySchedule) {
- // Handover: backoff restarts from the floor (or the latest hint),
- // not from the seed computed before the early polls began.
- inEarlySchedule = false;
- lastDelay = nextPollDelay(0, retryAfterMs, random);
+ const waitMs =
+ longPollActive && !scheduledThisRound
+ ? longPollWaitFor(deps.longPollWaitMs ?? 0, pollingDeadline - now())
+ : 0;
+ // A long-poll is sent right away: the server does the waiting.
+ let early: number | undefined;
+ if (waitMs === 0) {
+ early = earlyPollDelay({
+ pagesEstimate: deps.pagesEstimate,
+ elapsedMs: now() - startedAt,
+ pollCount,
+ fastPollCount,
+ retryAfterMs,
+ random,
+ });
+ if (early !== undefined && pollCount > 0) fastPollCount++;
+ if (early !== undefined) {
+ inEarlySchedule = true;
+ } else if (inEarlySchedule) {
+ // Handover: backoff restarts from the floor (or the latest hint),
+ // not from the seed computed before the early polls began.
+ inEarlySchedule = false;
+ lastDelay = nextPollDelay(0, retryAfterMs, random);
+ }
+ await sleep(
+ alignPollDelay(early ?? lastDelay, now(), deps.jobDeadlineAtMs),
+ meta.abort.asSignal(),
+ );
}
- await sleep(
- alignPollDelay(early ?? lastDelay, now(), deps.jobDeadlineAtMs),
- meta.abort.asSignal(),
- );
pollCount++;
+ const pollUrl =
+ waitMs > 0
+ ? `${baseUrl}/jobs/${scrapeId}?wait_ms=${waitMs}`
+ : `${baseUrl}/jobs/${scrapeId}`;
+ const sentAt = now();
let pollResp;
try {
- pollResp = await fetchImpl(`${baseUrl}/jobs/${scrapeId}`, {
+ pollResp = await fetchImpl(pollUrl, {
method: "GET",
headers: firePdfHeaders(),
signal: meta.abort.asSignal(),
@@ -122,6 +184,7 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
}
if (pollStatus === 410) {
+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();
firePdfAsyncPollCount.observe(pollCount);
const parsed = pollResponseSchema.safeParse(pollBody);
const status = parsed.success ? parsed.data.status : "expired";
@@ -134,6 +197,7 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
}
if (pollStatus === 502) {
+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();
firePdfAsyncPollCount.observe(pollCount);
firePdfAsyncCompletedTotal.labels("failed").inc();
failAsync(meta, "terminal_failed", {
@@ -162,6 +226,7 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
}
if (TERMINAL_STATUSES.has(parsed.data.status)) {
+ if (waitMs > 0) firePdfAsyncLongPollTotal.labels("terminal").inc();
firePdfAsyncPollCount.observe(pollCount);
firePdfAsyncCompletedTotal.labels(parsed.data.status).inc();
if (parsed.data.status !== "done") {
@@ -192,9 +257,19 @@ export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
);
}
+ if (waitMs > 0) {
+ const held = now() - sentAt >= waitMs * LONG_POLL_HELD_FRACTION;
+ firePdfAsyncLongPollTotal.labels(held ? "held" : "not_held").inc();
+ // Answered early without finishing: pause before the next wait_ms
+ // request, and after repeated early answers treat this server as not
+ // holding and use the regular schedule for the rest of the job.
+ earlyAnswers = held ? 0 : earlyAnswers + 1;
+ if (earlyAnswers >= LONG_POLL_MAX_EARLY_ANSWERS) longPollActive = false;
+ }
+
retryAfterMs = parsed.data.retry_after_ms;
// Backoff advances only while it is the schedule in use.
- if (early === undefined) {
+ if (waitMs === 0 && early === undefined) {
lastDelay = nextPollDelay(lastDelay, retryAfterMs, random);
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
index 7ca4c17c5..66ac5ee08 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/schema.ts
@@ -9,6 +9,23 @@ export const MIN_DEADLINE_MS = 5_000;
export const MAX_DEADLINE_MS = 30 * 60 * 1_000;
export const POLL_FLOOR_MS = 1_000;
export const POLL_CAP_MS = 5_000;
+// Long-poll (`wait_ms`). A wait shorter than LONG_POLL_MIN_WAIT_MS isn't
+// worth holding a connection for, so the last stretch before the polling
+// deadline runs on the regular schedule. A non-terminal answer that came
+// back in under LONG_POLL_HELD_FRACTION of the requested wait means the
+// server did not hold it (older build, or its wake-up path unavailable):
+// the job drops back to the regular schedule instead of re-polling at once.
+// LONG_POLL_MAX_WAIT_MS mirrors fire-pdf's default cap
+// (FIRE_PDF_JOBS_LONG_POLL_MAX_WAIT_MS): fire-pdf clamps longer waits, so
+// asking for more would only make a held answer look early. After
+// LONG_POLL_MAX_EARLY_ANSWERS early answers in a row the job stays on the
+// regular schedule; a single early answer is followed by one scheduled
+// pause, never an immediate re-poll.
+export const LONG_POLL_MIN_WAIT_MS = 1_000;
+export const LONG_POLL_MAX_WAIT_MS = 25_000;
+export const LONG_POLL_DEADLINE_SLACK_MS = 1_000;
+export const LONG_POLL_HELD_FRACTION = 0.5;
+export const LONG_POLL_MAX_EARLY_ANSWERS = 2;
export const POLL_TIMEOUT_BUFFER_MS = 30_000;
// An inline job's deadline sits this far inside the caller's window. The