Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/pdf): propagate fire-pdf pages_processed to billing (#3452)
Abimael Martell1b49d3fe58df
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.tsindex 6a1deeda6..c6c44ac32 100644--- a/apps/api/src/lib/gcs-pdf-cache.ts+++ b/apps/api/src/lib/gcs-pdf-cache.ts@@ -6,6 +6,15 @@ import { storage } from "./gcs-jobs";type PdfCacheProvider = "runpod" | "firepdf";+// Cache shape β markdown/html are required; pagesProcessed is optional so+// pre-existing entries (written before the field existed) round-trip cleanly+// and the caller can fall back to its own page-count signal on a stale hit.+type CachedPdfResult = {+ markdown: string;+ html: string;+ pagesProcessed?: number;+};+const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {runpod: "pdf-cache-v2/",firepdf: "pdf-cache-firepdf/",@@ -17,7 +26,7 @@ export function createPdfCacheKey(pdfContent: string | Buffer): string {export async function savePdfResultToCache(pdfContent: string,- result: { markdown: string; html: string },+ result: CachedPdfResult,provider: PdfCacheProvider = "runpod",): Promise<string | null> {try {@@ -74,7 +83,7 @@ export async function savePdfResultToCache(export async function getPdfResultFromCache(pdfContent: string,provider: PdfCacheProvider = "runpod",-): Promise<{ markdown: string; html: string } | null> {+): Promise<CachedPdfResult | null> {try {if (!config.GCS_BUCKET_NAME) {return null;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsnew file mode 100644index 000000000..f5068ca85--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -0,0 +1,43 @@+import { reconcilePageCountWithFirePdf } from "../firePDF";++describe("reconcilePageCountWithFirePdf", () => {+ it("uses fire-pdf's count when the upstream pass left it at 0", () => {+ // The original regression: processPdf threw "Invalid PDF structure" on a+ // malformed-but-still-renderable PDF, so effectivePageCount stayed 0.+ // fire-pdf processes it successfully and reports 15 pages. Billing must+ // see 15, not 0.+ expect(reconcilePageCountWithFirePdf(0, { pagesProcessed: 15 })).toBe(15);+ });++ it("never shrinks a count that an upstream pass already established", () => {+ // detectPdf / processPdf saw 20 pages; fire-pdf was called with+ // max_pages=10 and processed 10. We must keep 20 β fire-pdf's value+ // reflects its own cap, not the true PDF length.+ expect(reconcilePageCountWithFirePdf(20, { pagesProcessed: 10 })).toBe(20);+ });++ it("keeps current when both agree", () => {+ expect(reconcilePageCountWithFirePdf(15, { pagesProcessed: 15 })).toBe(15);+ });++ it("ignores undefined pagesProcessed (older fire-pdf or stale cache)", () => {+ // No signal β preserve whatever the upstream pass set, even if 0.+ expect(+ reconcilePageCountWithFirePdf(0, { pagesProcessed: undefined }),+ ).toBe(0);+ expect(reconcilePageCountWithFirePdf(7, {})).toBe(7);+ });++ it("ignores null/undefined result (fire-pdf didn't run)", () => {+ expect(reconcilePageCountWithFirePdf(7, null)).toBe(7);+ expect(reconcilePageCountWithFirePdf(7, undefined)).toBe(7);+ });++ it("treats fire-pdf's 0 as a real value (no special-casing)", () => {+ // If fire-pdf legitimately reports 0 (empty PDF that still rendered),+ // the max() semantic preserves whatever was already there. We only+ // skip when the field is *missing*, not when it's zero.+ expect(reconcilePageCountWithFirePdf(0, { pagesProcessed: 0 })).toBe(0);+ expect(reconcilePageCountWithFirePdf(5, { pagesProcessed: 0 })).toBe(5);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex 86fa38c70..3ea97d250 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -10,6 +10,36 @@ import {savePdfResultToCache,} from "../../../../lib/gcs-pdf-cache";+/**+ * Reconcile an existing page count with what fire-pdf reported.+ *+ * Used after `scrapePDFWithFirePDF` returns. The original bug: when Rust+ * extraction (`processPdf`) threw on a malformed-but-still-renderable PDF,+ * `effectivePageCount` stayed at 0; fire-pdf would then process the PDF+ * fine but its `pages_processed` value was dropped on the floor, so+ * `pdfMetadata.numPages` shipped as 0 and billing under-counted.+ *+ * Semantics:+ * - If fire-pdf didn't report a count (older fire-pdf builds, or stale+ * cache hits), keep the current value β no signal to act on.+ * - Otherwise take the max β never shrink a count that an upstream pass+ * (detectPdf / processPdf) already established. fire-pdf can be+ * called with `max_pages` capping its own processing below the true+ * PDF length, and the upstream count is the authoritative one when+ * both succeeded.+ *+ * Pure / synchronous so it's trivially unit-testable; the integration in+ * `index.ts` is just `effectivePageCount = reconcilePageCountWithFirePdf(...)`.+ */+export function reconcilePageCountWithFirePdf(+ current: number,+ firePdfResult: { pagesProcessed?: number } | null | undefined,+): number {+ const fromFirePdf = firePdfResult?.pagesProcessed;+ if (fromFirePdf === undefined) return current;+ return Math.max(current, fromFirePdf);+}+export async function scrapePDFWithFirePDF(meta: Meta,base64Content: string,@@ -25,7 +55,13 @@ export async function scrapePDFWithFirePDF(logger.info("Using cached FirePDF result", {scrapeId: meta.id,});- return cached;+ // Cache entries written before pagesProcessed existed don't carry+ // the field. Fall back to the caller's pagesProcessed argument so+ // billing on a stale hit doesn't silently regress to 0.+ return {+ ...cached,+ pagesProcessed: cached.pagesProcessed ?? pagesProcessed,+ };}} catch (error) {logger.warn("Error checking FirePDF cache, proceeding", { error });@@ -111,9 +147,10 @@ export async function scrapePDFWithFirePDF(perPageMs: pages ? Math.round(durationMs / pages) : undefined,});- const processorResult = {+ const processorResult: PDFProcessorResult & { markdown: string } = {markdown: resp.markdown,html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),+ pagesProcessed: pages,};if (!maxPages && !meta.internalOptions.zeroDataRetention) {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex e2b2adafd..ad4ef8583 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -33,7 +33,7 @@ import {} from "../../../../lib/native-logging";import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";import { scrapePDFWithRunPodMU } from "./runpodMU";-import { scrapePDFWithFirePDF } from "./firePDF";+import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";import { scrapePDFWithParsePDF } from "./pdfParse";import { captureExceptionWithZdrCheck } from "../../../../services/sentry";import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";@@ -411,6 +411,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {maxPages,effectivePageCount,);+ effectivePageCount = reconcilePageCountWithFirePdf(+ effectivePageCount,+ result,+ );} catch (error) {if (error instanceof RemoveFeatureError ||diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.tsindex 53d8ac167..11f206a2f 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts@@ -1,4 +1,15 @@-export type PDFProcessorResult = { html: string; markdown?: string };+export type PDFProcessorResult = {+ html: string;+ markdown?: string;+ /**+ * Pages the underlying engine actually processed for this request.+ * Currently populated only by fire-pdf (via OcrSuccessBody.pages_processed).+ * Optional because older fire-pdf builds and the runpodMU / pdf-parse+ * engines don't report it. Consumers must treat undefined as "no signal"+ * and fall back to whatever upstream metadata pass set.+ */+ pagesProcessed?: number;+};export type PdfMetadata = { numPages: number; title?: string };