Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): send FirePDF pages_estimate only when it is a positive count (#4885)

Abimael Martellae63ff03d2b0
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
index 4dbe60fcb..045822d32 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts
@@ -15,6 +15,7 @@ import {
} from "../fire-pdf/async";
import {
firePdfAsyncAbandonedTotal,
+ firePdfAsyncFallbackTotal,
firePdfAsyncSubmit503Total,
firePdfAsyncSubmitRetriesTotal,
} from "../fire-pdf/metrics";
@@ -262,6 +263,115 @@ describe("scrapePDFWithFirePDFAsync — deadline and submit lifecycle", () => {
).toBe(before + 1);
});
+ it.each([
+ ["a zero page count", 0],
+ ["no estimate at all", undefined],
+ ])(
+ "omits pages_estimate on an inline submit with %s so fire-pdf counts pages itself",
+ async (_name, pagesProcessed) => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: { scrape_id: "scrape-id-test", status: "done", lane: "fast" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: {
+ status: 200,
+ body: { markdown: "ok", pages_processed: 3 },
+ },
+ },
+ ]);
+
+ const result = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ pagesProcessed,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ );
+
+ expect(result.markdown).toBe("ok");
+ const options = (calls[0].body as { options?: Record<string, unknown> })
+ .options;
+ expect(options).not.toHaveProperty("pages_estimate");
+ },
+ );
+
+ it("sends a positive page count as pages_estimate", async () => {
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 200,
+ body: { scrape_id: "scrape-id-test", status: "done", lane: "fast" },
+ },
+ },
+ {
+ matchUrl: /\/jobs\/scrape-id-test\/result$/,
+ matchMethod: "GET",
+ response: { status: 200, body: { markdown: "ok", pages_processed: 7 } },
+ },
+ ]);
+
+ await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ 7,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ );
+
+ expect(
+ (calls[0].body as { options: Record<string, unknown> }).options
+ .pages_estimate,
+ ).toBe(7);
+ });
+
+ it("counts a submit 400 as an async fallback and keeps fire-pdf's code", async () => {
+ const before = await counterValue(firePdfAsyncFallbackTotal, {
+ reason: "http_400",
+ });
+ const { fetchImpl, calls } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: {
+ status: 400,
+ body: {
+ error: "invalid_pages_estimate",
+ message: "options.pages_estimate must be a positive integer",
+ },
+ },
+ },
+ ]);
+
+ const error = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },
+ ).catch(e => e);
+
+ expect(error).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(error.reason).toBe("http_400");
+ expect(error.extra.code).toBe("invalid_pages_estimate");
+ expect(calls).toHaveLength(1);
+ expect(
+ await counterValue(firePdfAsyncFallbackTotal, { reason: "http_400" }),
+ ).toBe(before + 1);
+ });
+
it("does not retry fire-pdf's own 503 codes", async () => {
const before = await counterValue(firePdfAsyncSubmit503Total, {
code: "admission_rejected",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
index 917819e79..e9b114dbb 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts
@@ -87,6 +87,7 @@ export type SubmitRetryTrigger =
export type AbandonedPhase = "submit" | "poll" | "result";
export type FallbackReason =
+ | "http_400"
| "http_401"
| "http_404"
| "http_410"
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index fa0951bba..38a924820 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -254,7 +254,13 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
body: json,
},
);
- throw new Error("fire-pdf async POST /jobs validation error");
+ // Counted like every other exit from the async path, with fire-pdf's
+ // validation code, so 400s appear in the fallback metric.
+ const code =
+ typeof (json as { error?: unknown } | null)?.error === "string"
+ ? (json as { error: string }).error
+ : "unattributed";
+ failAsync(meta, "http_400", { code });
}
if (status !== 200 && status !== 202) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
index 661dafa79..6e7e06027 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -160,9 +160,15 @@ export function buildFirePdfJobOptions(args: {
pageMarkers: boolean;
}): Record<string, unknown> {
return {
- ...(args.pagesProcessed !== undefined && {
- pages_estimate: args.pagesProcessed,
- }),
+ // fire-pdf accepts only a positive integer here. When the router has no
+ // page count (0), the field is left out and fire-pdf counts the pages
+ // itself on inline submits. By-reference submits require a positive
+ // count and check for one before they get here.
+ ...(args.pagesProcessed !== undefined &&
+ Number.isInteger(args.pagesProcessed) &&
+ args.pagesProcessed > 0 && {
+ pages_estimate: args.pagesProcessed,
+ }),
...(args.maxPages !== undefined && { max_pages: args.maxPages }),
...(args.mode !== undefined && { mode: args.mode }),
...(args.includePageMarkdown && { include_page_markdown: true }),