Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): send FirePDF pages_estimate only when it is a positive count (#4885)
Abimael Martellae63ff03d2b0
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.tsindex 4dbe60fcb..045822d32 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsyncDeadline.test.ts@@ -15,6 +15,7 @@ import {} from "../fire-pdf/async";import {firePdfAsyncAbandonedTotal,+ firePdfAsyncFallbackTotal,firePdfAsyncSubmit503Total,firePdfAsyncSubmitRetriesTotal,} from "../fire-pdf/metrics";@@ -262,6 +263,115 @@ describe("scrapePDFWithFirePDFAsync — deadline and submit lifecycle", () => {).toBe(before + 1);});+ it.each([+ ["a zero page count", 0],+ ["no estimate at all", undefined],+ ])(+ "omits pages_estimate on an inline submit with %s so fire-pdf counts pages itself",+ async (_name, pagesProcessed) => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: { scrape_id: "scrape-id-test", status: "done", lane: "fast" },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: {+ status: 200,+ body: { markdown: "ok", pages_processed: 3 },+ },+ },+ ]);++ const result = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ pagesProcessed,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ );++ expect(result.markdown).toBe("ok");+ const options = (calls[0].body as { options?: Record<string, unknown> })+ .options;+ expect(options).not.toHaveProperty("pages_estimate");+ },+ );++ it("sends a positive page count as pages_estimate", async () => {+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 200,+ body: { scrape_id: "scrape-id-test", status: "done", lane: "fast" },+ },+ },+ {+ matchUrl: /\/jobs\/scrape-id-test\/result$/,+ matchMethod: "GET",+ response: { status: 200, body: { markdown: "ok", pages_processed: 7 } },+ },+ ]);++ await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ 7,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ );++ expect(+ (calls[0].body as { options: Record<string, unknown> }).options+ .pages_estimate,+ ).toBe(7);+ });++ it("counts a submit 400 as an async fallback and keeps fire-pdf's code", async () => {+ const before = await counterValue(firePdfAsyncFallbackTotal, {+ reason: "http_400",+ });+ const { fetchImpl, calls } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: {+ status: 400,+ body: {+ error: "invalid_pages_estimate",+ message: "options.pages_estimate must be a positive integer",+ },+ },+ },+ ]);++ const error = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ undefined,+ { fetchImpl, fallbackImpl: vi.fn(), sleepImpl: noopSleep },+ ).catch(e => e);++ expect(error).toBeInstanceOf(FirePdfAsyncFailure);+ expect(error.reason).toBe("http_400");+ expect(error.extra.code).toBe("invalid_pages_estimate");+ expect(calls).toHaveLength(1);+ expect(+ await counterValue(firePdfAsyncFallbackTotal, { reason: "http_400" }),+ ).toBe(before + 1);+ });+it("does not retry fire-pdf's own 503 codes", async () => {const before = await counterValue(firePdfAsyncSubmit503Total, {code: "admission_rejected",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.tsindex 917819e79..e9b114dbb 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/metrics.ts@@ -87,6 +87,7 @@ export type SubmitRetryTrigger =export type AbandonedPhase = "submit" | "poll" | "result";export type FallbackReason =+ | "http_400"| "http_401"| "http_404"| "http_410"diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex fa0951bba..38a924820 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -254,7 +254,13 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {body: json,},);- throw new Error("fire-pdf async POST /jobs validation error");+ // Counted like every other exit from the async path, with fire-pdf's+ // validation code, so 400s appear in the fallback metric.+ const code =+ typeof (json as { error?: unknown } | null)?.error === "string"+ ? (json as { error: string }).error+ : "unattributed";+ failAsync(meta, "http_400", { code });}if (status !== 200 && status !== 202) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex 661dafa79..6e7e06027 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -160,9 +160,15 @@ export function buildFirePdfJobOptions(args: {pageMarkers: boolean;}): Record<string, unknown> {return {- ...(args.pagesProcessed !== undefined && {- pages_estimate: args.pagesProcessed,- }),+ // fire-pdf accepts only a positive integer here. When the router has no+ // page count (0), the field is left out and fire-pdf counts the pages+ // itself on inline submits. By-reference submits require a positive+ // count and check for one before they get here.+ ...(args.pagesProcessed !== undefined &&+ Number.isInteger(args.pagesProcessed) &&+ args.pagesProcessed > 0 && {+ pages_estimate: args.pagesProcessed,+ }),...(args.maxPages !== undefined && { max_pages: args.maxPages }),...(args.mode !== undefined && { mode: args.mode }),...(args.includePageMarkdown && { include_page_markdown: true }),