Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api/pdf): propagate fire-pdf pages_processed to billing (#3452)

Abimael Martell1b49d3fe58df
diff --git a/apps/api/src/lib/gcs-pdf-cache.ts b/apps/api/src/lib/gcs-pdf-cache.ts
index 6a1deeda6..c6c44ac32 100644
--- a/apps/api/src/lib/gcs-pdf-cache.ts
+++ b/apps/api/src/lib/gcs-pdf-cache.ts
@@ -6,6 +6,15 @@ import { storage } from "./gcs-jobs";
type PdfCacheProvider = "runpod" | "firepdf";
+// Cache shape β€” markdown/html are required; pagesProcessed is optional so
+// pre-existing entries (written before the field existed) round-trip cleanly
+// and the caller can fall back to its own page-count signal on a stale hit.
+type CachedPdfResult = {
+ markdown: string;
+ html: string;
+ pagesProcessed?: number;
+};
+
const PROVIDER_PREFIXES: Record<PdfCacheProvider, string> = {
runpod: "pdf-cache-v2/",
firepdf: "pdf-cache-firepdf/",
@@ -17,7 +26,7 @@ export function createPdfCacheKey(pdfContent: string | Buffer): string {
export async function savePdfResultToCache(
pdfContent: string,
- result: { markdown: string; html: string },
+ result: CachedPdfResult,
provider: PdfCacheProvider = "runpod",
): Promise<string | null> {
try {
@@ -74,7 +83,7 @@ export async function savePdfResultToCache(
export async function getPdfResultFromCache(
pdfContent: string,
provider: PdfCacheProvider = "runpod",
-): Promise<{ markdown: string; html: string } | null> {
+): Promise<CachedPdfResult | null> {
try {
if (!config.GCS_BUCKET_NAME) {
return null;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
new file mode 100644
index 000000000..f5068ca85
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -0,0 +1,43 @@
+import { reconcilePageCountWithFirePdf } from "../firePDF";
+
+describe("reconcilePageCountWithFirePdf", () => {
+ it("uses fire-pdf's count when the upstream pass left it at 0", () => {
+ // The original regression: processPdf threw "Invalid PDF structure" on a
+ // malformed-but-still-renderable PDF, so effectivePageCount stayed 0.
+ // fire-pdf processes it successfully and reports 15 pages. Billing must
+ // see 15, not 0.
+ expect(reconcilePageCountWithFirePdf(0, { pagesProcessed: 15 })).toBe(15);
+ });
+
+ it("never shrinks a count that an upstream pass already established", () => {
+ // detectPdf / processPdf saw 20 pages; fire-pdf was called with
+ // max_pages=10 and processed 10. We must keep 20 β€” fire-pdf's value
+ // reflects its own cap, not the true PDF length.
+ expect(reconcilePageCountWithFirePdf(20, { pagesProcessed: 10 })).toBe(20);
+ });
+
+ it("keeps current when both agree", () => {
+ expect(reconcilePageCountWithFirePdf(15, { pagesProcessed: 15 })).toBe(15);
+ });
+
+ it("ignores undefined pagesProcessed (older fire-pdf or stale cache)", () => {
+ // No signal β€” preserve whatever the upstream pass set, even if 0.
+ expect(
+ reconcilePageCountWithFirePdf(0, { pagesProcessed: undefined }),
+ ).toBe(0);
+ expect(reconcilePageCountWithFirePdf(7, {})).toBe(7);
+ });
+
+ it("ignores null/undefined result (fire-pdf didn't run)", () => {
+ expect(reconcilePageCountWithFirePdf(7, null)).toBe(7);
+ expect(reconcilePageCountWithFirePdf(7, undefined)).toBe(7);
+ });
+
+ it("treats fire-pdf's 0 as a real value (no special-casing)", () => {
+ // If fire-pdf legitimately reports 0 (empty PDF that still rendered),
+ // the max() semantic preserves whatever was already there. We only
+ // skip when the field is *missing*, not when it's zero.
+ expect(reconcilePageCountWithFirePdf(0, { pagesProcessed: 0 })).toBe(0);
+ expect(reconcilePageCountWithFirePdf(5, { pagesProcessed: 0 })).toBe(5);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index 86fa38c70..3ea97d250 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -10,6 +10,36 @@ import {
savePdfResultToCache,
} from "../../../../lib/gcs-pdf-cache";
+/**
+ * Reconcile an existing page count with what fire-pdf reported.
+ *
+ * Used after `scrapePDFWithFirePDF` returns. The original bug: when Rust
+ * extraction (`processPdf`) threw on a malformed-but-still-renderable PDF,
+ * `effectivePageCount` stayed at 0; fire-pdf would then process the PDF
+ * fine but its `pages_processed` value was dropped on the floor, so
+ * `pdfMetadata.numPages` shipped as 0 and billing under-counted.
+ *
+ * Semantics:
+ * - If fire-pdf didn't report a count (older fire-pdf builds, or stale
+ * cache hits), keep the current value β€” no signal to act on.
+ * - Otherwise take the max β€” never shrink a count that an upstream pass
+ * (detectPdf / processPdf) already established. fire-pdf can be
+ * called with `max_pages` capping its own processing below the true
+ * PDF length, and the upstream count is the authoritative one when
+ * both succeeded.
+ *
+ * Pure / synchronous so it's trivially unit-testable; the integration in
+ * `index.ts` is just `effectivePageCount = reconcilePageCountWithFirePdf(...)`.
+ */
+export function reconcilePageCountWithFirePdf(
+ current: number,
+ firePdfResult: { pagesProcessed?: number } | null | undefined,
+): number {
+ const fromFirePdf = firePdfResult?.pagesProcessed;
+ if (fromFirePdf === undefined) return current;
+ return Math.max(current, fromFirePdf);
+}
+
export async function scrapePDFWithFirePDF(
meta: Meta,
base64Content: string,
@@ -25,7 +55,13 @@ export async function scrapePDFWithFirePDF(
logger.info("Using cached FirePDF result", {
scrapeId: meta.id,
});
- return cached;
+ // Cache entries written before pagesProcessed existed don't carry
+ // the field. Fall back to the caller's pagesProcessed argument so
+ // billing on a stale hit doesn't silently regress to 0.
+ return {
+ ...cached,
+ pagesProcessed: cached.pagesProcessed ?? pagesProcessed,
+ };
}
} catch (error) {
logger.warn("Error checking FirePDF cache, proceeding", { error });
@@ -111,9 +147,10 @@ export async function scrapePDFWithFirePDF(
perPageMs: pages ? Math.round(durationMs / pages) : undefined,
});
- const processorResult = {
+ const processorResult: PDFProcessorResult & { markdown: string } = {
markdown: resp.markdown,
html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),
+ pagesProcessed: pages,
};
if (!maxPages && !meta.internalOptions.zeroDataRetention) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index e2b2adafd..ad4ef8583 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -33,7 +33,7 @@ import {
} from "../../../../lib/native-logging";
import { withSpan, setSpanAttributes } from "../../../../lib/otel-tracer";
import { scrapePDFWithRunPodMU } from "./runpodMU";
-import { scrapePDFWithFirePDF } from "./firePDF";
+import { reconcilePageCountWithFirePdf, scrapePDFWithFirePDF } from "./firePDF";
import { scrapePDFWithParsePDF } from "./pdfParse";
import { captureExceptionWithZdrCheck } from "../../../../services/sentry";
import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";
@@ -411,6 +411,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
maxPages,
effectivePageCount,
);
+ effectivePageCount = reconcilePageCountWithFirePdf(
+ effectivePageCount,
+ result,
+ );
} catch (error) {
if (
error instanceof RemoveFeatureError ||
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
index 53d8ac167..11f206a2f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/types.ts
@@ -1,4 +1,15 @@
-export type PDFProcessorResult = { html: string; markdown?: string };
+export type PDFProcessorResult = {
+ html: string;
+ markdown?: string;
+ /**
+ * Pages the underlying engine actually processed for this request.
+ * Currently populated only by fire-pdf (via OcrSuccessBody.pages_processed).
+ * Optional because older fire-pdf builds and the runpodMU / pdf-parse
+ * engines don't report it. Consumers must treat undefined as "no signal"
+ * and fall back to whatever upstream metadata pass set.
+ */
+ pagesProcessed?: number;
+};
export type PdfMetadata = { numPages: number; title?: string };