Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): count every FirePDF transport decision and keep fire-pdf's 503 code (#4819)
Abimael Martell4cf431dc1a17
diff --git a/apps/api/src/scraper/scrapeURL/engines/image/index.ts b/apps/api/src/scraper/scrapeURL/engines/image/index.tsindex 9ad468510..a081d7fec 100644--- a/apps/api/src/scraper/scrapeURL/engines/image/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/image/index.ts@@ -5,6 +5,7 @@ import { EngineUnsuccessfulError, UnsupportedFileError } from "../../error";import { readFile, stat, unlink } from "node:fs/promises";import { useFireEngine } from "../fire-engine/available";import { scrapePDFWithFirePDF } from "../pdf/firePDF";+import { recordFirePdfRoute } from "../pdf/fire-pdf/routing";import { FIRE_PDF_INLINE_HARD_MAX_FILE_SIZE } from "../pdf/types";import {imageExtensionFromUrlPath,@@ -140,6 +141,13 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {const base64Content = buffer.toString("base64");+ recordFirePdfRoute(meta, {+ sourceKind: "image",+ path: "sync",+ reason: "no_async_route",+ features: "none",+ });+let result: Awaited<ReturnType<typeof scrapePDFWithFirePDF>>;try {result = await scrapePDFWithFirePDF(diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.tsindex ceef41988..e29f8dfdf 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts@@ -15,6 +15,7 @@ import {} from "../fire-pdf/async";import {firePdfAsyncAbandonedTotal,+ firePdfAsyncSubmit503Total,firePdfAsyncSubmitRetriesTotal,} from "../fire-pdf/metrics";import { AbortManagerThrownError } from "../../../lib/abortManager";@@ -208,6 +209,9 @@ describe("scrapePDFWithFirePDFAsync β deadline and submit lifecycle", () => {});it("does not retry fire-pdf's own 503 codes", async () => {+ const before = await counterValue(firePdfAsyncSubmit503Total, {+ code: "admission_rejected",+ });const { fetchImpl, calls } = makeFetchFromSequence([{matchUrl: /\/jobs$/,@@ -233,7 +237,45 @@ describe("scrapePDFWithFirePDFAsync β deadline and submit lifecycle", () => {expect(error).toBeInstanceOf(FirePdfAsyncFailure);expect(error.reason).toBe("http_503");+ expect(error.extra.code).toBe("admission_rejected");expect(calls).toHaveLength(1);+ expect(+ await counterValue(firePdfAsyncSubmit503Total, {+ code: "admission_rejected",+ }),+ ).toBe(before + 1);+ });++ it("labels a 503 without a fire-pdf code as unattributed once the retry also fails", async () => {+ const before = await counterValue(firePdfAsyncSubmit503Total, {+ code: "unattributed",+ });+ const unattributed = {+ matchUrl: /\/jobs$/,+ matchMethod: "POST" as const,+ response: { status: 503, body: { detail: "upstream overloaded" } },+ };+ const { fetchImpl, calls } = makeFetchFromSequence([+ unattributed,+ unattributed,+ ]);++ const error = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ ).catch(e => e);++ expect(error).toBeInstanceOf(FirePdfAsyncFailure);+ expect(error.reason).toBe("http_503");+ expect(error.extra.code).toBe("unattributed");+ expect(calls.filter(c => c.method === "POST")).toHaveLength(2);+ expect(+ await counterValue(firePdfAsyncSubmit503Total, { code: "unattributed" }),+ ).toBe(before + 1);});it("retries the submit once on a transport failure, then proceeds", async () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.tsindex ea413a987..fbaa5da05 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncRouting.test.ts@@ -1,9 +1,16 @@import { config } from "../../../../../config";+import {+ firePdfRouteDecisionsTotal,+ firePdfRouteRemainingSeconds,+} from "../fire-pdf/metrics";import {decideFirePdfAsyncRoute,deterministicPercentage,FIRE_PDF_ASYNC_MIN_REMAINING_MS,+ firePdfFeaturesLabel,+ recordFirePdfRoute,} from "../fire-pdf/routing";+import { counterValue, makeMeta } from "./firePDFAsyncFixtures";import {computeByReferenceDeadlineMs,computeDeadlineMs,@@ -198,3 +205,147 @@ describe("FirePDF async transport helpers", () => {}});});++describe("FirePDF route decision telemetry", () => {+ it("labels page-aware features in a fixed order", () => {+ const none = { pageMarkdown: false, blocks: false, pageMarkers: false };+ expect(firePdfFeaturesLabel(none)).toBe("none");+ expect(firePdfFeaturesLabel({ ...none, pageMarkdown: true })).toBe("pages");+ expect(+ firePdfFeaturesLabel({+ pageMarkdown: true,+ blocks: true,+ pageMarkers: true,+ }),+ ).toBe("pages+blocks+markers");+ expect(+ firePdfFeaturesLabel({ ...none, pageMarkers: true, blocks: true }),+ ).toBe("blocks+markers");+ });++ it("counts a sync decision with its reason, logs it, and records the time left", async () => {+ const labels = {+ source_kind: "pdf",+ path: "sync",+ reason: "zdr",+ features: "pages",+ zdr: "true",+ };+ const before = await counterValue(firePdfRouteDecisionsTotal, labels);+ const syncPdf = (+ vals: Awaited<+ ReturnType<typeof firePdfRouteRemainingSeconds.get>+ >["values"],+ metricName: string,+ le?: number,+ ) =>+ vals.find(+ v =>+ v.metricName === metricName &&+ v.labels.path === "sync" &&+ v.labels.source_kind === "pdf" &&+ (le === undefined || (v.labels as Record<string, unknown>).le === le),+ )?.value ?? 0;+ const BUCKET = "firecrawl_fire_pdf_route_remaining_seconds_bucket";+ const COUNT = "firecrawl_fire_pdf_route_remaining_seconds_count";+ const { values: histBefore } = await firePdfRouteRemainingSeconds.get();++ const meta = makeMeta({+ internalOptions: { zeroDataRetention: true, teamId: "team-x" },+ });+ meta.abort.scrapeTimeout.mockReturnValue(15_000);+ recordFirePdfRoute(meta, {+ sourceKind: "pdf",+ path: "sync",+ reason: "zdr",+ features: "pages",+ });++ expect(await counterValue(firePdfRouteDecisionsTotal, labels)).toBe(+ before + 1,+ );+ const { values: histAfter } = await firePdfRouteRemainingSeconds.get();+ // 15 s lands in the le=15 bucket, not the one below it.+ expect(syncPdf(histAfter, BUCKET, 15)).toBe(+ syncPdf(histBefore, BUCKET, 15) + 1,+ );+ expect(syncPdf(histAfter, BUCKET, 10)).toBe(+ syncPdf(histBefore, BUCKET, 10),+ );+ expect(syncPdf(histAfter, COUNT)).toBe(syncPdf(histBefore, COUNT) + 1);+ expect(meta.logger.info).toHaveBeenCalledWith(+ "Routing FirePDF request to sync /ocr",+ expect.objectContaining({+ event: "fire_pdf_sync_routed",+ reason: "zdr",+ source_kind: "pdf",+ remaining_ms: 15_000,+ }),+ );+ });++ it("uses the remaining time captured before the attempt when one is passed", async () => {+ const bucket = async (le: number) =>+ (await firePdfRouteRemainingSeconds.get()).values.find(+ v =>+ v.metricName ===+ "firecrawl_fire_pdf_route_remaining_seconds_bucket" &&+ v.labels.path === "sync" &&+ v.labels.source_kind === "pdf" &&+ (v.labels as Record<string, unknown>).le === le,+ )?.value ?? 0;+ const le30Before = await bucket(30);+ const le45Before = await bucket(45);++ const meta = makeMeta();+ meta.abort.scrapeTimeout.mockReturnValue(2_000);+ recordFirePdfRoute(meta, {+ sourceKind: "pdf",+ path: "sync",+ reason: "outside_percentage",+ features: "none",+ remainingMs: 45_000,+ });+ expect(meta.logger.info).toHaveBeenCalledWith(+ "Routing FirePDF request to sync /ocr",+ expect.objectContaining({ remaining_ms: 45_000 }),+ );+ // 45 s, not the 2 s the scrape has left now: le=45 moves, le=30 doesn't.+ expect(await bucket(45)).toBe(le45Before + 1);+ expect(await bucket(30)).toBe(le30Before);+ });++ it("counts async decisions without a second log line and skips the histogram when there is no deadline", async () => {+ const labels = {+ source_kind: "pdf",+ path: "async",+ reason: "by_reference",+ features: "none",+ zdr: "false",+ };+ const before = await counterValue(firePdfRouteDecisionsTotal, labels);+ const { values: histBefore } = await firePdfRouteRemainingSeconds.get();+ const countOf = (vals: typeof histBefore) =>+ vals.find(+ v =>+ v.metricName === "firecrawl_fire_pdf_route_remaining_seconds_count" &&+ v.labels.path === "async",+ )?.value ?? 0;++ const meta = makeMeta();+ meta.abort.scrapeTimeout.mockReturnValue(undefined);+ recordFirePdfRoute(meta, {+ sourceKind: "pdf",+ path: "async",+ reason: "by_reference",+ features: "none",+ });++ expect(await counterValue(firePdfRouteDecisionsTotal, labels)).toBe(+ before + 1,+ );+ const { values: histAfter } = await firePdfRouteRemainingSeconds.get();+ expect(countOf(histAfter)).toBe(countOf(histBefore));+ expect(meta.logger.info).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex b4621e21c..917819e79 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -36,6 +36,25 @@ export const firePdfAsyncFallbackTotal = new Counter({labelNames: ["reason"],});+export const firePdfRouteDecisionsTotal = new Counter({+ name: "firecrawl_fire_pdf_route_decisions_total",+ help: "Transport chosen for a request's first FirePDF attempt: path=sync (POST /ocr) or async (POST /jobs), by routing reason",+ labelNames: ["source_kind", "path", "reason", "features", "zdr"],+});++export const firePdfRouteRemainingSeconds = new Histogram({+ name: "firecrawl_fire_pdf_route_remaining_seconds",+ help: "Caller time left on the scrape when the FirePDF transport was chosen",+ labelNames: ["source_kind", "path"],+ buckets: [5, 10, 15, 20, 30, 45, 60, 90, 120, 180, 300, 600],+});++export const firePdfAsyncSubmit503Total = new Counter({+ name: "firecrawl_fire_pdf_async_submit_503_total",+ help: "fire-pdf POST /jobs 503s that ended the async attempt, by the error code in the response body (unattributed when the body carried none)",+ labelNames: ["code"],+});+export const firePdfAsyncTotalDurationSeconds = new Histogram({name: "firecrawl_fire_pdf_async_total_duration_seconds",help: "End-to-end duration from 'decide to use async' to 'result available'",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.tsindex ae4c26e23..44d918c7b 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/routing.ts@@ -1,4 +1,10 @@import { createHash } from "node:crypto";+import type { Meta } from "../../..";+import {+ firePdfRouteDecisionsTotal,+ firePdfRouteRemainingSeconds,+} from "./metrics";+import type { FirePdfSourceKind } from "./request-metadata";import { MIN_ASYNC_CALLER_WINDOW_MS } from "./schema";export const FIRE_PDF_ASYNC_MIN_REMAINING_MS = MIN_ASYNC_CALLER_WINDOW_MS;@@ -96,3 +102,78 @@ export function decideFirePdfAsyncRoute(}return { enabled: false, reason: "outside_percentage" };}++/** Why a request's first FirePDF attempt took the transport it did: the+ * async cohort decision for inline PDFs, plus the two routes that never+ * consult it (large PDFs always go by reference; images have no async+ * route yet). */+type FirePdfRouteReason =+ | FirePdfAsyncRouteReason+ | "by_reference"+ | "no_async_route";++type FirePdfRoute = {+ sourceKind: FirePdfSourceKind;+ path: "sync" | "async";+ reason: FirePdfRouteReason;+ features: string;+ /** Caller time left when the transport was chosen. Pass it when the+ * decision is recorded after the attempt ran; defaults to now. */+ remainingMs?: number;+};++/** Stable label for the page-aware options a request asked for, e.g.+ * "none" or "pages+markers". At most 8 values. */+export function firePdfFeaturesLabel(features: {+ pageMarkdown: boolean;+ blocks: boolean;+ pageMarkers: boolean;+}): string {+ const parts = [+ features.pageMarkdown && "pages",+ features.blocks && "blocks",+ features.pageMarkers && "markers",+ ].filter(Boolean);+ return parts.length > 0 ? parts.join("+") : "none";+}++/**+ * Counts every first-attempt transport decision, sync and async alike, so+ * the sync remainder can be attributed (ZDR, short deadline, outside the+ * cohort, images) without reconstructing it from logs. Labels only: no URL+ * or content, so it is safe for ZDR requests. Async decisions are already+ * logged by the caller (`fire_pdf_async_routed`); sync ones are logged here.+ */+export function recordFirePdfRoute(meta: Meta, route: FirePdfRoute): void {+ const zdr = meta.internalOptions.zeroDataRetention ?? false;+ firePdfRouteDecisionsTotal+ .labels(+ route.sourceKind,+ route.path,+ route.reason,+ route.features,+ String(zdr),+ )+ .inc();++ const remainingMs =+ "remainingMs" in route ? route.remainingMs : meta.abort.scrapeTimeout();+ if (remainingMs !== undefined) {+ firePdfRouteRemainingSeconds+ .labels(route.sourceKind, route.path)+ .observe(Math.max(0, remainingMs) / 1000);+ }++ if (route.path === "sync") {+ meta.logger.info("Routing FirePDF request to sync /ocr", {+ method: route.sourceKind === "image" ? "scrapeImage" : "scrapePDF",+ event: "fire_pdf_sync_routed",+ source_kind: route.sourceKind,+ reason: route.reason,+ features: route.features,+ remaining_ms: remainingMs,+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ });+ }+}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 28f7a1aea..fa0951bba 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -4,6 +4,7 @@ import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import { buildFirePdfRequestMetadata } from "./request-metadata";import {+ firePdfAsyncSubmit503Total,firePdfAsyncSubmitRetriesTotal,firePdfAsyncSubmittedTotal,type SubmitRetryTrigger,@@ -223,7 +224,14 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {if (status === 413) failAsync(meta, "http_413");if (status === 429) failAsync(meta, "http_429");if (status === 502) failAsync(meta, "http_502", { body: json });- if (status === 503) failAsync(meta, "http_503");+ if (status === 503) {+ // Keep fire-pdf's own code (page_markdown_not_ready, admission_rejected,+ // ...) so the 503 bucket can be split without the request body.+ const parsed503 = firePdfSubmit503BodySchema.safeParse(json);+ const code = parsed503.success ? parsed503.data.error : "unattributed";+ firePdfAsyncSubmit503Total.labels(code).inc();+ failAsync(meta, "http_503", { code });+ }if (status === 409) {meta.logger.error(diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex f3875d2c5..b43c37a94 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -55,7 +55,11 @@ import {mineruDiverted,} from "./fire-pdf/by-reference";import { runFirePdfByReferenceAttempt } from "./fire-pdf/by-reference-flow";-import { decideFirePdfAsyncRoute } from "./fire-pdf/routing";+import {+ decideFirePdfAsyncRoute,+ firePdfFeaturesLabel,+ recordFirePdfRoute,+} from "./fire-pdf/routing";import { scrapePDFWithParsePDF } from "./pdfParse";import { toPublicBlocks } from "./blocks";import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";@@ -538,18 +542,42 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {// A null return means the input never made it into the fire-pdf// bucket: fall through to the legacy chain, whose oversized-skip// warning below still fires (pre-by-reference behavior).- const byRefResult = await runFirePdfByReferenceAttempt({- meta,- tempFilePath,- fileSizeBytes,- pagesEstimate: effectivePageCount,- mode,- maxPages,- includePageMarkdown,- includeBlocks,- pageMarkers,- });+ // Counted only once the attempt used the async transport (a+ // result, or a throw after placement): a null return falls+ // through to the inline chain, which records its own decision,+ // so counting here too would double-count the request.+ const byRefRoute = {+ sourceKind: "pdf" as const,+ path: "async" as const,+ reason: "by_reference" as const,+ features: firePdfFeaturesLabel({+ pageMarkdown: includePageMarkdown,+ blocks: includeBlocks,+ pageMarkers: pageMarkers,+ }),+ remainingMs: meta.abort.scrapeTimeout(),+ };+ let byRefResult: Awaited<+ ReturnType<typeof runFirePdfByReferenceAttempt>+ >;+ try {+ byRefResult = await runFirePdfByReferenceAttempt({+ meta,+ tempFilePath,+ fileSizeBytes,+ pagesEstimate: effectivePageCount,+ mode,+ maxPages,+ includePageMarkdown,+ includeBlocks,+ pageMarkers,+ });+ } catch (error) {+ recordFirePdfRoute(meta, byRefRoute);+ throw error;+ }if (byRefResult) {+ recordFirePdfRoute(meta, byRefRoute);result = byRefResult;effectivePageCount = reconcilePageCountWithFirePdf(effectivePageCount,@@ -648,6 +676,16 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {bulkOriginPercentage: config.FIRE_PDF_ASYNC_BULK_ORIGIN_PERCENT,});const useAsync = asyncDecision.enabled;+ recordFirePdfRoute(meta, {+ sourceKind: "pdf",+ path: useAsync ? "async" : "sync",+ reason: asyncDecision.reason,+ features: firePdfFeaturesLabel({+ pageMarkdown: includePageMarkdown,+ blocks: includeBlocks,+ pageMarkers: pageMarkers,+ }),+ });if (useAsync) {meta.logger.info("Routing FirePDF request to async jobs", {method: "scrapePDF",