Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): count every FirePDF transport decision and keep fire-pdf's 503 code (#4819)

Abimael Martell4cf431dc1a17
diff --git a/apps/api/src/scraper/scrapeURL/engines/image/index.ts b/apps/api/src/scraper/scrapeURL/engines/image/index.ts
index 9ad468510..a081d7fec 100644
--- a/apps/api/src/scraper/scrapeURL/engines/image/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/image/index.ts
@@ -5,6 +5,7 @@ import { EngineUnsuccessfulError, UnsupportedFileError } from "../../error";
import { readFile, stat, unlink } from "node:fs/promises";
import { useFireEngine } from "../fire-engine/available";
import { scrapePDFWithFirePDF } from "../pdf/firePDF";
+import { recordFirePdfRoute } from "../pdf/fire-pdf/routing";
import { FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE } from "../pdf/types";
import {
imageExtensionFromUrlPath,
@@ -140,6 +141,13 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {
const base64Content = buffer.toString("base64");
+ recordFirePdfRoute(meta, {
+ sourceKind: "image",
+ path: "sync",
+ reason: "no_async_route",
+ features: "none",
+ });
+
let result: Awaited<ReturnType<typeof scrapePDFWithFirePDF>>;
try {
result = await scrapePDFWithFirePDF(
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
index ceef41988..e29f8dfdf 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
@@ -15,6 +15,7 @@ import {
} from "../fire-pdf/async";
import {
firePdfAsyncAbandonedTotal,
+ firePdfAsyncSubmit503Total,
firePdfAsyncSubmitRetriesTotal,
} from "../fire-pdf/metrics";
import { AbortManagerThrownError } from "../../../lib/abortManager";
@@ -208,6 +209,9 @@ describe("scrapePDFWithFirePDFAsync β€” deadline and submit lifecycle", () => {
});
it("does not retry fire-pdf's own 503 codes", async () => {
+ const before = await counterValue(firePdfAsyncSubmit503Total, {
+ code: "admission_rejected",
+ });
const { fetchImpl, calls } = makeFetchFromSequence([
{
matchUrl: /\/jobs$/,
@@ -233,7 +237,45 @@ describe("scrapePDFWithFirePDFAsync β€” deadline and submit lifecycle", () => {
expect(error).toBeInstanceOf(FirePdfAsyncFailure);
expect(error.reason).toBe("http_503");
+ expect(error.extra.code).toBe("admission_rejected");
expect(calls).toHaveLength(1);
+ expect(
+ await counterValue(firePdfAsyncSubmit503Total, {
+ code: "admission_rejected",
+ }),
+ ).toBe(before + 1);
+ });
+
+ it("labels a 503 without a fire-pdf code as unattributed once the retry also fails", async () => {
+ const before = await counterValue(firePdfAsyncSubmit503Total, {
+ code: "unattributed",
+ });
+ const unattributed = {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST" as const,
+ response: { status: 503, body: { detail: "upstream overloaded" } },
+ };
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ unattributed,
+ unattributed,
+ ]);
+
+ const error = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(error.reason).toBe("http_503");
+ expect(error.extra.code).toBe("unattributed");
+ expect(calls.filter(c => c.method === "POST")).toHaveLength(2);
+ expect(
+ await counterValue(firePdfAsyncSubmit503Total, { code: "unattributed" }),
+ ).toBe(before + 1);
});
it("retries the submit once on a transport failure, then proceeds", async () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
index ea413a987..fbaa5da05 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts
@@ -1,9 +1,16 @@
import { config } from "../../../../../config";
+import {
+ firePdfRouteDecisionsTotal,
+ firePdfRouteRemainingSeconds,
+} from "../fire-pdf/metrics";
import {
decideFirePdfAsyncRoute,
deterministicPercentage,
FIRE_PDF_ASYNC_MIN_REMAINING_MS,
+ firePdfFeaturesLabel,
+ recordFirePdfRoute,
} from "../fire-pdf/routing";
+import { counterValue, makeMeta } from "./firePDFAsyncFixtures";
import {
computeByReferenceDeadlineMs,
computeDeadlineMs,
@@ -198,3 +205,147 @@ describe("FirePDF async transport helpers", () => {
}
});
});
+
+describe("FirePDF route decision telemetry", () => {
+ it("labels page-aware features in a fixed order", () => {
+ const none = { pageMarkdown: false, blocks: false, pageMarkers: false };
+ expect(firePdfFeaturesLabel(none)).toBe("none");
+ expect(firePdfFeaturesLabel({ ...none, pageMarkdown: true })).toBe("pages");
+ expect(
+ firePdfFeaturesLabel({
+ pageMarkdown: true,
+ blocks: true,
+ pageMarkers: true,
+ }),
+ ).toBe("pages+blocks+markers");
+ expect(
+ firePdfFeaturesLabel({ ...none, pageMarkers: true, blocks: true }),
+ ).toBe("blocks+markers");
+ });
+
+ it("counts a sync decision with its reason, logs it, and records the time left", async () => {
+ const labels = {
+ source_kind: "pdf",
+ path: "sync",
+ reason: "zdr",
+ features: "pages",
+ zdr: "true",
+ };
+ const before = await counterValue(firePdfRouteDecisionsTotal, labels);
+ const syncPdf = (
+ vals: Awaited<
+ ReturnType<typeof firePdfRouteRemainingSeconds.get>
+ >["values"],
+ metricName: string,
+ le?: number,
+ ) =>
+ vals.find(
+ v =>
+ v.metricName === metricName &&
+ v.labels.path === "sync" &&
+ v.labels.source_kind === "pdf" &&
+ (le === undefined || (v.labels as Record<string, unknown>).le === le),
+ )?.value ?? 0;
+ const BUCKET = "firecrawl_fire_pdf_route_remaining_seconds_bucket";
+ const COUNT = "firecrawl_fire_pdf_route_remaining_seconds_count";
+ const { values: histBefore } = await firePdfRouteRemainingSeconds.get();
+
+ const meta = makeMeta({
+ internalOptions: { zeroDataRetention: true, teamId: "team-x" },
+ });
+ meta.abort.scrapeTimeout.mockReturnValue(15_000);
+ recordFirePdfRoute(meta, {
+ sourceKind: "pdf",
+ path: "sync",
+ reason: "zdr",
+ features: "pages",
+ });
+
+ expect(await counterValue(firePdfRouteDecisionsTotal, labels)).toBe(
+ before + 1,
+ );
+ const { values: histAfter } = await firePdfRouteRemainingSeconds.get();
+ // 15 s lands in the le=15 bucket, not the one below it.
+ expect(syncPdf(histAfter, BUCKET, 15)).toBe(
+ syncPdf(histBefore, BUCKET, 15) + 1,
+ );
+ expect(syncPdf(histAfter, BUCKET, 10)).toBe(
+ syncPdf(histBefore, BUCKET, 10),
+ );
+ expect(syncPdf(histAfter, COUNT)).toBe(syncPdf(histBefore, COUNT) + 1);
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "Routing FirePDF request to sync /ocr",
+ expect.objectContaining({
+ event: "fire_pdf_sync_routed",
+ reason: "zdr",
+ source_kind: "pdf",
+ remaining_ms: 15_000,
+ }),
+ );
+ });
+
+ it("uses the remaining time captured before the attempt when one is passed", async () => {
+ const bucket = async (le: number) =>
+ (await firePdfRouteRemainingSeconds.get()).values.find(
+ v =>
+ v.metricName ===
+ "firecrawl_fire_pdf_route_remaining_seconds_bucket" &&
+ v.labels.path === "sync" &&
+ v.labels.source_kind === "pdf" &&
+ (v.labels as Record<string, unknown>).le === le,
+ )?.value ?? 0;
+ const le30Before = await bucket(30);
+ const le45Before = await bucket(45);
+
+ const meta = makeMeta();
+ meta.abort.scrapeTimeout.mockReturnValue(2_000);
+ recordFirePdfRoute(meta, {
+ sourceKind: "pdf",
+ path: "sync",
+ reason: "outside_percentage",
+ features: "none",
+ remainingMs: 45_000,
+ });
+ expect(meta.logger.info).toHaveBeenCalledWith(
+ "Routing FirePDF request to sync /ocr",
+ expect.objectContaining({ remaining_ms: 45_000 }),
+ );
+ // 45 s, not the 2 s the scrape has left now: le=45 moves, le=30 doesn't.
+ expect(await bucket(45)).toBe(le45Before + 1);
+ expect(await bucket(30)).toBe(le30Before);
+ });
+
+ it("counts async decisions without a second log line and skips the histogram when there is no deadline", async () => {
+ const labels = {
+ source_kind: "pdf",
+ path: "async",
+ reason: "by_reference",
+ features: "none",
+ zdr: "false",
+ };
+ const before = await counterValue(firePdfRouteDecisionsTotal, labels);
+ const { values: histBefore } = await firePdfRouteRemainingSeconds.get();
+ const countOf = (vals: typeof histBefore) =>
+ vals.find(
+ v =>
+ v.metricName === "firecrawl_fire_pdf_route_remaining_seconds_count" &&
+ v.labels.path === "async",
+ )?.value ?? 0;
+
+ const meta = makeMeta();
+ meta.abort.scrapeTimeout.mockReturnValue(undefined);
+ recordFirePdfRoute(meta, {
+ sourceKind: "pdf",
+ path: "async",
+ reason: "by_reference",
+ features: "none",
+ });
+
+ expect(await counterValue(firePdfRouteDecisionsTotal, labels)).toBe(
+ before + 1,
+ );
+ const { values: histAfter } = await firePdfRouteRemainingSeconds.get();
+ expect(countOf(histAfter)).toBe(countOf(histBefore));
+ expect(meta.logger.info).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
index b4621e21c..917819e79 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -36,6 +36,25 @@ export const firePdfAsyncFallbackTotal = new Counter({
labelNames: ["reason"],
});
+export const firePdfRouteDecisionsTotal = new Counter({
+ name: "firecrawl_fire_pdf_route_decisions_total",
+ help: "Transport chosen for a request's first FirePDF attempt: path=sync (POST /ocr) or async (POST /jobs), by routing reason",
+ labelNames: ["source_kind", "path", "reason", "features", "zdr"],
+});
+
+export const firePdfRouteRemainingSeconds = new Histogram({
+ name: "firecrawl_fire_pdf_route_remaining_seconds",
+ help: "Caller time left on the scrape when the FirePDF transport was chosen",
+ labelNames: ["source_kind", "path"],
+ buckets: [5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 300, 600],
+});
+
+export const firePdfAsyncSubmit503Total = new Counter({
+ name: "firecrawl_fire_pdf_async_submit_503_total",
+ help: "fire-pdf POST /jobs 503s that ended the async attempt, by the error code in the response body (unattributed when the body carried none)",
+ labelNames: ["code"],
+});
+
export const firePdfAsyncTotalDurationSeconds = new Histogram({
name: "firecrawl_fire_pdf_async_total_duration_seconds",
help: "End-to-end duration from 'decide to use async' to 'result available'",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts
index ae4c26e23..44d918c7b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts
@@ -1,4 +1,10 @@
import { createHash } from "node:crypto";
+import type { Meta } from "../../..";
+import {
+ firePdfRouteDecisionsTotal,
+ firePdfRouteRemainingSeconds,
+} from "./metrics";
+import type { FirePdfSourceKind } from "./request-metadata";
import { MIN_ASYNC_CALLER_WINDOW_MS } from "./schema";
export const FIRE_PDF_ASYNC_MIN_REMAINING_MS = MIN_ASYNC_CALLER_WINDOW_MS;
@@ -96,3 +102,78 @@ export function decideFirePdfAsyncRoute(
}
return { enabled: false, reason: "outside_percentage" };
}
+
+/** Why a request's first FirePDF attempt took the transport it did: the
+ * async cohort decision for inline PDFs, plus the two routes that never
+ * consult it (large PDFs always go by reference; images have no async
+ * route yet). */
+type FirePdfRouteReason =
+ | FirePdfAsyncRouteReason
+ | "by_reference"
+ | "no_async_route";
+
+type FirePdfRoute = {
+ sourceKind: FirePdfSourceKind;
+ path: "sync" | "async";
+ reason: FirePdfRouteReason;
+ features: string;
+ /** Caller time left when the transport was chosen. Pass it when the
+ * decision is recorded after the attempt ran; defaults to now. */
+ remainingMs?: number;
+};
+
+/** Stable label for the page-aware options a request asked for, e.g.
+ * "none" or "pages+markers". At most 8 values. */
+export function firePdfFeaturesLabel(features: {
+ pageMarkdown: boolean;
+ blocks: boolean;
+ pageMarkers: boolean;
+}): string {
+ const parts = [
+ features.pageMarkdown && "pages",
+ features.blocks && "blocks",
+ features.pageMarkers && "markers",
+ ].filter(Boolean);
+ return parts.length > 0 ? parts.join("+") : "none";
+}
+
+/**
+ * Counts every first-attempt transport decision, sync and async alike, so
+ * the sync remainder can be attributed (ZDR, short deadline, outside the
+ * cohort, images) without reconstructing it from logs. Labels only: no URL
+ * or content, so it is safe for ZDR requests. Async decisions are already
+ * logged by the caller (`fire_pdf_async_routed`); sync ones are logged here.
+ */
+export function recordFirePdfRoute(meta: Meta, route: FirePdfRoute): void {
+ const zdr = meta.internalOptions.zeroDataRetention ?? false;
+ firePdfRouteDecisionsTotal
+ .labels(
+ route.sourceKind,
+ route.path,
+ route.reason,
+ route.features,
+ String(zdr),
+ )
+ .inc();
+
+ const remainingMs =
+ "remainingMs" in route ? route.remainingMs : meta.abort.scrapeTimeout();
+ if (remainingMs !== undefined) {
+ firePdfRouteRemainingSeconds
+ .labels(route.sourceKind, route.path)
+ .observe(Math.max(0, remainingMs) / 1000);
+ }
+
+ if (route.path === "sync") {
+ meta.logger.info("Routing FirePDF request to sync /ocr", {
+ method: route.sourceKind === "image" ? "scrapeImage" : "scrapePDF",
+ event: "fire_pdf_sync_routed",
+ source_kind: route.sourceKind,
+ reason: route.reason,
+ features: route.features,
+ remaining_ms: remainingMs,
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ });
+ }
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 28f7a1aea..fa0951bba 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -4,6 +4,7 @@ import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import { buildFirePdfRequestMetadata } from "./request-metadata";
import {
+ firePdfAsyncSubmit503Total,
firePdfAsyncSubmitRetriesTotal,
firePdfAsyncSubmittedTotal,
type SubmitRetryTrigger,
@@ -223,7 +224,14 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
if (status === 413) failAsync(meta, "http_413");
if (status === 429) failAsync(meta, "http_429");
if (status === 502) failAsync(meta, "http_502", { body: json });
- if (status === 503) failAsync(meta, "http_503");
+ if (status === 503) {
+ // Keep fire-pdf's own code (page_markdown_not_ready, admission_rejected,
+ // ...) so the 503 bucket can be split without the request body.
+ const parsed503 = firePdfSubmit503BodySchema.safeParse(json);
+ const code = parsed503.success ? parsed503.data.error : "unattributed";
+ firePdfAsyncSubmit503Total.labels(code).inc();
+ failAsync(meta, "http_503", { code });
+ }
if (status === 409) {
meta.logger.error(
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index f3875d2c5..b43c37a94 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -55,7 +55,11 @@ import {
mineruDiverted,
} from "./fire-pdf/by-reference";
import { runFirePdfByReferenceAttempt } from "./fire-pdf/by-reference-flow";
-import { decideFirePdfAsyncRoute } from "./fire-pdf/routing";
+import {
+ decideFirePdfAsyncRoute,
+ firePdfFeaturesLabel,
+ recordFirePdfRoute,
+} from "./fire-pdf/routing";
import { scrapePDFWithParsePDF } from "./pdfParse";
import { toPublicBlocks } from "./blocks";
import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";
@@ -538,18 +542,42 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
// A null return means the input never made it into the fire-pdf
// bucket: fall through to the legacy chain, whose oversized-skip
// warning below still fires (pre-by-reference behavior).
- const byRefResult = await runFirePdfByReferenceAttempt({
- meta,
- tempFilePath,
- fileSizeBytes,
- pagesEstimate: effectivePageCount,
- mode,
- maxPages,
- includePageMarkdown,
- includeBlocks,
- pageMarkers,
- });
+ // Counted only once the attempt used the async transport (a
+ // result, or a throw after placement): a null return falls
+ // through to the inline chain, which records its own decision,
+ // so counting here too would double-count the request.
+ const byRefRoute = {
+ sourceKind: "pdf" as const,
+ path: "async" as const,
+ reason: "by_reference" as const,
+ features: firePdfFeaturesLabel({
+ pageMarkdown: includePageMarkdown,
+ blocks: includeBlocks,
+ pageMarkers: pageMarkers,
+ }),
+ remainingMs: meta.abort.scrapeTimeout(),
+ };
+ let byRefResult: Awaited<
+ ReturnType<typeof runFirePdfByReferenceAttempt>
+ >;
+ try {
+ byRefResult = await runFirePdfByReferenceAttempt({
+ meta,
+ tempFilePath,
+ fileSizeBytes,
+ pagesEstimate: effectivePageCount,
+ mode,
+ maxPages,
+ includePageMarkdown,
+ includeBlocks,
+ pageMarkers,
+ });
+ } catch (error) {
+ recordFirePdfRoute(meta, byRefRoute);
+ throw error;
+ }
if (byRefResult) {
+ recordFirePdfRoute(meta, byRefRoute);
result = byRefResult;
effectivePageCount = reconcilePageCountWithFirePdf(
effectivePageCount,
@@ -648,6 +676,16 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
bulkOriginPercentage: config.FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT,
});
const useAsync = asyncDecision.enabled;
+ recordFirePdfRoute(meta, {
+ sourceKind: "pdf",
+ path: useAsync ? "async" : "sync",
+ reason: asyncDecision.reason,
+ features: firePdfFeaturesLabel({
+ pageMarkdown: includePageMarkdown,
+ blocks: includeBlocks,
+ pageMarkers: pageMarkers,
+ }),
+ });
if (useAsync) {
meta.logger.info("Routing FirePDF request to async jobs", {
method: "scrapePDF",