Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { Meta } from "../..";
import { config } from "../../../../config";
import { robustFetch } from "../../lib/fetch";
import { z } from "zod";
import type { PDFProcessorResult } from "./types";
import type { PDFMode } from "../../../../controllers/v2/types";
import { safeMarkdownToHtml } from "./markdownToHtml";
import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";
import { isUsOnly, US_DATA_REGION } from "../../../../lib/us-only";
import { isReservedCapacity } from "../../../../lib/reserved-capacity";
import {
maybeSaveResult,
provenanceFromResponse,
tryGetCached,
} from "./fire-pdf/cache";
import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";
import {
buildFirePdfRequestMetadata,
buildFirePdfTrainingCapture,
firePdfTableFormat,
type FirePdfSourceKind,
} from "./fire-pdf/request-metadata";
import {
firePdfReservedRefused,
firePdfReservedCapacityRefusal,
firePdfRegionRefused,
firePdfUsOnlyRefusal,
} from "./fire-pdf/data-region";
/**
* Reconcile an existing page count with what fire-pdf reported.
*
* Used after `scrapePDFWithFirePDF` returns. The original bug: when Rust
* extraction (`processPdf`) threw on a malformed-but-still-renderable PDF,
* `effectivePageCount` stayed at 0; fire-pdf would then process the PDF
* fine but its `pages_processed` value was dropped on the floor, so
* `pdfMetadata.numPages` shipped as 0 and billing under-counted.
*
* Semantics:
* - If fire-pdf didn't report a count (older fire-pdf builds, or stale
* cache hits), keep the current value — no signal to act on.
* - Otherwise take the max — never shrink a count that an upstream pass
* (detectPdf / processPdf) already established. fire-pdf can be
* called with `max_pages` capping its own processing below the true
* PDF length, and the upstream count is the authoritative one when
* both succeeded.
*
* Pure / synchronous so it's trivially unit-testable; the integration in
* `index.ts` is just `effectivePageCount = reconcilePageCountWithFirePdf(...)`.
*/
export function reconcilePageCountWithFirePdf(
current: number,
firePdfResult: { pagesProcessed?: number } | null | undefined,
): number {
const fromFirePdf = firePdfResult?.pagesProcessed;
if (fromFirePdf === undefined) return current;
return Math.max(current, fromFirePdf);
}
export async function scrapePDFWithFirePDF(
meta: Meta,
base64Content: string,
maxPages?: number,
pagesProcessed?: number,
mode?: PDFMode,
includePageMarkdown = false,
includeBlocks = false,
pageMarkers = false,
sourceKind: FirePdfSourceKind = "pdf",
): Promise<PDFProcessorResult> {
const logger = meta.logger;
// Cache layout:
// - `ocr` mode reads/writes a dedicated `…-ocr.json` bucket. ocr
// requests explicitly want forced layout-mode OCR, so they must
// not be served a base-cache entry that was written by `auto`.
// - `auto` (and legacy undefined-mode) reads/writes the base
// `firepdf-<sha>.json` bucket — same key main has always used,
// so existing entries keep working. As a free upgrade, auto also
// reads the ocr bucket as a fallback: if some prior `ocr` run
// already produced markdown for this PDF, reuse it rather than
// running fire-pdf again.
// - `fast` is bypassed entirely (hard cost ceiling — must fail on
// scanned PDFs, not serve a cached OCR result).
// - `page_markers` rewrites the document markdown itself (inter-page
// `<!-- page N -->` separators), so marker requests read/write a
// fully disjoint `…markers…` variant family — a base-variant entry
// must never be served for a marker request and vice versa. See
// cacheKeyShape in fire-pdf/cache.ts.
const cacheable =
mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;
const cached = cacheable
? await tryGetCached(
meta,
base64Content,
mode,
maxPages,
pagesProcessed,
includePageMarkdown,
includeBlocks,
pageMarkers,
)
: null;
if (cached) return cached;
meta.abort.throwIfAborted();
const startedAt = Date.now();
logger.info("FirePDF started", {
scrapeId: meta.id,
url: meta.rewrittenUrl ?? meta.url,
maxPages,
pagesProcessed,
});
const zdr = meta.internalOptions.zeroDataRetention === true;
const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
const reservedCapacity = isReservedCapacity(
meta.internalOptions.teamFlags ?? undefined,
);
const pdfSha256 = createPdfCacheKey(base64Content);
const tableFormat = firePdfTableFormat(meta);
// Explicit deadline contract with fire-pdf (mirrors mineru-api):
// timeout — remaining scrape-tier budget in ms (from AbortManager)
// created_at — epoch ms when we handed the budget over to fire-pdf
//
// fire-pdf computes remaining = timeout - (now - created_at) and can
// return 503 if the budget is spent. Previously it only saw the abort
// signal from the HTTP connection, which it didn't observe — so work
// kept running past the caller's timeout and the user got a late
// failure instead of a fast deadline-exceeded response.
//
// scrapeTimeout() returns undefined if no scrape-tier deadline is set
// (e.g., internal tests, CLI). Don't send timeout in that case so
// fire-pdf applies its own default.
const fireScrapeTimeout = meta.abort.scrapeTimeout();
const deadlineFields: { timeout?: number; created_at?: number } = {};
if (fireScrapeTimeout !== undefined && fireScrapeTimeout > 0) {
deadlineFields.timeout = Math.floor(fireScrapeTimeout);
deadlineFields.created_at = Date.now();
}
const resp = await robustFetch({
url: `${config.FIRE_PDF_BASE_URL}/ocr`,
method: "POST",
headers: config.FIRE_PDF_API_KEY
? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }
: undefined,
body: {
pdf: base64Content,
scrape_id: meta.id,
...(maxPages !== undefined && { max_pages: maxPages }),
...(mode !== undefined && { mode }),
...(includePageMarkdown && { include_page_markdown: true }),
...(includeBlocks && { include_blocks: true }),
...(pageMarkers && { page_markers: true }),
...(tableFormat && { tableFormat }),
// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
// these as optional — older fire-pdf builds will ignore unknown fields.
team_id: meta.internalOptions.teamId,
...(meta.internalOptions.crawlId && {
crawl_id: meta.internalOptions.crawlId,
}),
...buildFirePdfRequestMetadata(meta, sourceKind),
pdf_sha256: pdfSha256,
source: "firecrawl",
zdr,
training_capture: buildFirePdfTrainingCapture(meta),
...deadlineFields,
...(usOnly && { data_region: US_DATA_REGION }),
...(reservedCapacity && { reserved: true }),
},
logger,
schema: z.object({
markdown: z.string(),
failed_pages: z.array(z.number()).nullable(),
partial_pages: z.array(z.number()).nullable().optional(),
pages_processed: z.number().optional(),
pages: firePdfPagesSchema,
blocks: firePdfBlocksSchema,
// fire-pdf's stamp (generation, build, stages, quality). Taken raw
// and parsed separately (provenanceFromResponse) so a stamp this
// build cannot read never fails the scrape; a missing stamp means an
// older fire-pdf build.
provenance: z.unknown().optional(),
// Echo of an honored page_markers request. Markers are baked into
// `markdown` and their absence is not reliably detectable there (a
// single-page or fully-stitched document legitimately has none), so
// the echo is the only proof the fire-pdf build understood the
// option — older builds ignore unknown request fields and omit it.
page_markers: z.literal(true).optional(),
data_region_applied: z.unknown().optional(),
reserved_applied: z.unknown().optional(),
}),
mock: meta.mock,
abort: meta.abort.asSignal(),
}).catch(error => {
const body = (error as { cause?: { response?: { body?: unknown } } })?.cause
?.response?.body;
if (usOnly && firePdfRegionRefused(body)) {
throw firePdfUsOnlyRefusal(meta, "region_unavailable", sourceKind);
}
if (reservedCapacity && firePdfReservedRefused(body)) {
throw firePdfReservedCapacityRefusal(
meta,
"reserved_unavailable",
sourceKind,
);
}
throw error;
});
// A build that ignores the pin answers without the echo; its result may
// have come from GPU capacity outside the US.
if (usOnly && resp.data_region_applied !== US_DATA_REGION) {
throw firePdfUsOnlyRefusal(meta, "region_unconfirmed", sourceKind);
}
// Likewise, without the reserved echo the result may have come from other
// capacity.
if (reservedCapacity && resp.reserved_applied !== true) {
throw firePdfReservedCapacityRefusal(
meta,
"reserved_unconfirmed",
sourceKind,
);
}
const durationMs = Date.now() - startedAt;
if (includePageMarkdown && resp.pages === undefined) {
throw new Error(
"FirePDF response did not include requested physical page markdown",
);
}
if (includeBlocks && resp.blocks === undefined) {
throw new Error("FirePDF response did not include requested typed blocks");
}
if (pageMarkers && resp.page_markers !== true) {
// Without the echo, the markdown is ordinary unmarked output; caching
// it under a marker variant would silently poison the marker cache.
throw new Error(
"FirePDF response did not acknowledge requested page markers",
);
}
const pages = resp.pages_processed ?? pagesProcessed;
const provenance = provenanceFromResponse(resp.provenance, logger, {
scrapeId: meta.id,
cacheKey: pdfSha256,
});
logger.info("FirePDF completed", {
scrapeId: meta.id,
url: meta.rewrittenUrl ?? meta.url,
durationMs,
markdownLength: resp.markdown.length,
failedPages: resp.failed_pages,
partialPages: resp.partial_pages ?? null,
pagesProcessed: pages,
perPageMs: pages ? Math.round(durationMs / pages) : undefined,
// The content-cache key and the producer, so a report can be turned
// into keys to purge and a result can be tied to a fire-pdf build.
cacheKey: pdfSha256,
generation: provenance?.generation ?? "unknown",
buildSha: provenance?.build_sha ?? "unknown",
});
const processorResult: PDFProcessorResult & { markdown: string } = {
markdown: resp.markdown,
html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),
pagesProcessed: pages,
...(resp.pages ? { pageMarkdown: resp.pages } : {}),
...(resp.blocks ? { blocks: resp.blocks } : {}),
};
if (cacheable) {
await maybeSaveResult({
meta,
base64Content,
mode,
maxPages,
includePageMarkdown,
includeBlocks,
pageMarkers,
result: processorResult,
provenance,
failedPages: resp.failed_pages,
});
}
return processorResult;
}