Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

10.9 KB
Raw Blame
import { Meta } from "../..";
import { config } from "../../../../config";
import { robustFetch } from "../../lib/fetch";
import { z } from "zod";
import type { PDFProcessorResult } from "./types";
import type { PDFMode } from "../../../../controllers/v2/types";
import { safeMarkdownToHtml } from "./markdownToHtml";
import { createPdfCacheKey } from "../../../../lib/gcs-pdf-cache";
import { isUsOnly, US_DATA_REGION } from "../../../../lib/us-only";
import { isReservedCapacity } from "../../../../lib/reserved-capacity";
import {
  maybeSaveResult,
  provenanceFromResponse,
  tryGetCached,
} from "./fire-pdf/cache";
import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";
import {
  buildFirePdfRequestMetadata,
  buildFirePdfTrainingCapture,
  firePdfTableFormat,
  type FirePdfSourceKind,
} from "./fire-pdf/request-metadata";
import {
  firePdfReservedRefused,
  firePdfReservedCapacityRefusal,
  firePdfRegionRefused,
  firePdfUsOnlyRefusal,
} from "./fire-pdf/data-region";

/**
 * Reconcile an existing page count with what fire-pdf reported.
 *
 * Used after `scrapePDFWithFirePDF` returns. The original bug: when Rust
 * extraction (`processPdf`) threw on a malformed-but-still-renderable PDF,
 * `effectivePageCount` stayed at 0; fire-pdf would then process the PDF
 * fine but its `pages_processed` value was dropped on the floor, so
 * `pdfMetadata.numPages` shipped as 0 and billing under-counted.
 *
 * Semantics:
 *   - If fire-pdf didn't report a count (older fire-pdf builds, or stale
 *     cache hits), keep the current value — no signal to act on.
 *   - Otherwise take the max — never shrink a count that an upstream pass
 *     (detectPdf / processPdf) already established. fire-pdf can be
 *     called with `max_pages` capping its own processing below the true
 *     PDF length, and the upstream count is the authoritative one when
 *     both succeeded.
 *
 * Pure / synchronous so it's trivially unit-testable; the integration in
 * `index.ts` is just `effectivePageCount = reconcilePageCountWithFirePdf(...)`.
 */
export function reconcilePageCountWithFirePdf(
  current: number,
  firePdfResult: { pagesProcessed?: number } | null | undefined,
): number {
  const fromFirePdf = firePdfResult?.pagesProcessed;
  if (fromFirePdf === undefined) return current;
  return Math.max(current, fromFirePdf);
}

export async function scrapePDFWithFirePDF(
  meta: Meta,
  base64Content: string,
  maxPages?: number,
  pagesProcessed?: number,
  mode?: PDFMode,
  includePageMarkdown = false,
  includeBlocks = false,
  pageMarkers = false,
  sourceKind: FirePdfSourceKind = "pdf",
): Promise<PDFProcessorResult> {
  const logger = meta.logger;

  // Cache layout:
  //   - `ocr` mode reads/writes a dedicated `…-ocr.json` bucket. ocr
  //     requests explicitly want forced layout-mode OCR, so they must
  //     not be served a base-cache entry that was written by `auto`.
  //   - `auto` (and legacy undefined-mode) reads/writes the base
  //     `firepdf-<sha>.json` bucket — same key main has always used,
  //     so existing entries keep working. As a free upgrade, auto also
  //     reads the ocr bucket as a fallback: if some prior `ocr` run
  //     already produced markdown for this PDF, reuse it rather than
  //     running fire-pdf again.
  //   - `fast` is bypassed entirely (hard cost ceiling — must fail on
  //     scanned PDFs, not serve a cached OCR result).
  //   - `page_markers` rewrites the document markdown itself (inter-page
  //     `<!-- page N -->` separators), so marker requests read/write a
  //     fully disjoint `…markers…` variant family — a base-variant entry
  //     must never be served for a marker request and vice versa. See
  //     cacheKeyShape in fire-pdf/cache.ts.
  const cacheable =
    mode !== "fast" && !maxPages && !meta.internalOptions.zeroDataRetention;
  const cached = cacheable
    ? await tryGetCached(
        meta,
        base64Content,
        mode,
        maxPages,
        pagesProcessed,
        includePageMarkdown,
        includeBlocks,
        pageMarkers,
      )
    : null;
  if (cached) return cached;

  meta.abort.throwIfAborted();

  const startedAt = Date.now();

  logger.info("FirePDF started", {
    scrapeId: meta.id,
    url: meta.rewrittenUrl ?? meta.url,
    maxPages,
    pagesProcessed,
  });

  const zdr = meta.internalOptions.zeroDataRetention === true;
  const usOnly = isUsOnly(meta.internalOptions.teamFlags ?? undefined);
  const reservedCapacity = isReservedCapacity(
    meta.internalOptions.teamFlags ?? undefined,
  );
  const pdfSha256 = createPdfCacheKey(base64Content);
  const tableFormat = firePdfTableFormat(meta);

  // Explicit deadline contract with fire-pdf (mirrors mineru-api):
  //   timeout    — remaining scrape-tier budget in ms (from AbortManager)
  //   created_at — epoch ms when we handed the budget over to fire-pdf
  //
  // fire-pdf computes remaining = timeout - (now - created_at) and can
  // return 503 if the budget is spent. Previously it only saw the abort
  // signal from the HTTP connection, which it didn't observe — so work
  // kept running past the caller's timeout and the user got a late
  // failure instead of a fast deadline-exceeded response.
  //
  // scrapeTimeout() returns undefined if no scrape-tier deadline is set
  // (e.g., internal tests, CLI). Don't send timeout in that case so
  // fire-pdf applies its own default.
  const fireScrapeTimeout = meta.abort.scrapeTimeout();
  const deadlineFields: { timeout?: number; created_at?: number } = {};
  if (fireScrapeTimeout !== undefined && fireScrapeTimeout > 0) {
    deadlineFields.timeout = Math.floor(fireScrapeTimeout);
    deadlineFields.created_at = Date.now();
  }

  const resp = await robustFetch({
    url: `${config.FIRE_PDF_BASE_URL}/ocr`,
    method: "POST",
    headers: config.FIRE_PDF_API_KEY
      ? { Authorization: `Bearer ${config.FIRE_PDF_API_KEY}` }
      : undefined,
    body: {
      pdf: base64Content,
      scrape_id: meta.id,
      ...(maxPages !== undefined && { max_pages: maxPages }),
      ...(mode !== undefined && { mode }),
      ...(includePageMarkdown && { include_page_markdown: true }),
      ...(includeBlocks && { include_blocks: true }),
      ...(pageMarkers && { page_markers: true }),
      ...(tableFormat && { tableFormat }),
      // Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
      // these as optional — older fire-pdf builds will ignore unknown fields.
      team_id: meta.internalOptions.teamId,
      ...(meta.internalOptions.crawlId && {
        crawl_id: meta.internalOptions.crawlId,
      }),
      ...buildFirePdfRequestMetadata(meta, sourceKind),
      pdf_sha256: pdfSha256,
      source: "firecrawl",
      zdr,
      training_capture: buildFirePdfTrainingCapture(meta),
      ...deadlineFields,
      ...(usOnly && { data_region: US_DATA_REGION }),
      ...(reservedCapacity && { reserved: true }),
    },
    logger,
    schema: z.object({
      markdown: z.string(),
      failed_pages: z.array(z.number()).nullable(),
      partial_pages: z.array(z.number()).nullable().optional(),
      pages_processed: z.number().optional(),
      pages: firePdfPagesSchema,
      blocks: firePdfBlocksSchema,
      // fire-pdf's stamp (generation, build, stages, quality). Taken raw
      // and parsed separately (provenanceFromResponse) so a stamp this
      // build cannot read never fails the scrape; a missing stamp means an
      // older fire-pdf build.
      provenance: z.unknown().optional(),
      // Echo of an honored page_markers request. Markers are baked into
      // `markdown` and their absence is not reliably detectable there (a
      // single-page or fully-stitched document legitimately has none), so
      // the echo is the only proof the fire-pdf build understood the
      // option — older builds ignore unknown request fields and omit it.
      page_markers: z.literal(true).optional(),
      data_region_applied: z.unknown().optional(),
      reserved_applied: z.unknown().optional(),
    }),
    mock: meta.mock,
    abort: meta.abort.asSignal(),
  }).catch(error => {
    const body = (error as { cause?: { response?: { body?: unknown } } })?.cause
      ?.response?.body;
    if (usOnly && firePdfRegionRefused(body)) {
      throw firePdfUsOnlyRefusal(meta, "region_unavailable", sourceKind);
    }
    if (reservedCapacity && firePdfReservedRefused(body)) {
      throw firePdfReservedCapacityRefusal(
        meta,
        "reserved_unavailable",
        sourceKind,
      );
    }
    throw error;
  });
  // A build that ignores the pin answers without the echo; its result may
  // have come from GPU capacity outside the US.
  if (usOnly && resp.data_region_applied !== US_DATA_REGION) {
    throw firePdfUsOnlyRefusal(meta, "region_unconfirmed", sourceKind);
  }
  // Likewise, without the reserved echo the result may have come from other
  // capacity.
  if (reservedCapacity && resp.reserved_applied !== true) {
    throw firePdfReservedCapacityRefusal(
      meta,
      "reserved_unconfirmed",
      sourceKind,
    );
  }

  const durationMs = Date.now() - startedAt;
  if (includePageMarkdown && resp.pages === undefined) {
    throw new Error(
      "FirePDF response did not include requested physical page markdown",
    );
  }
  if (includeBlocks && resp.blocks === undefined) {
    throw new Error("FirePDF response did not include requested typed blocks");
  }
  if (pageMarkers && resp.page_markers !== true) {
    // Without the echo, the markdown is ordinary unmarked output; caching
    // it under a marker variant would silently poison the marker cache.
    throw new Error(
      "FirePDF response did not acknowledge requested page markers",
    );
  }
  const pages = resp.pages_processed ?? pagesProcessed;
  const provenance = provenanceFromResponse(resp.provenance, logger, {
    scrapeId: meta.id,
    cacheKey: pdfSha256,
  });

  logger.info("FirePDF completed", {
    scrapeId: meta.id,
    url: meta.rewrittenUrl ?? meta.url,
    durationMs,
    markdownLength: resp.markdown.length,
    failedPages: resp.failed_pages,
    partialPages: resp.partial_pages ?? null,
    pagesProcessed: pages,
    perPageMs: pages ? Math.round(durationMs / pages) : undefined,
    // The content-cache key and the producer, so a report can be turned
    // into keys to purge and a result can be tied to a fire-pdf build.
    cacheKey: pdfSha256,
    generation: provenance?.generation ?? "unknown",
    buildSha: provenance?.build_sha ?? "unknown",
  });

  const processorResult: PDFProcessorResult & { markdown: string } = {
    markdown: resp.markdown,
    html: await safeMarkdownToHtml(resp.markdown, logger, meta.id),
    pagesProcessed: pages,
    ...(resp.pages ? { pageMarkdown: resp.pages } : {}),
    ...(resp.blocks ? { blocks: resp.blocks } : {}),
  };

  if (cacheable) {
    await maybeSaveResult({
      meta,
      base64Content,
      mode,
      maxPages,
      includePageMarkdown,
      includeBlocks,
      pageMarkers,
      result: processorResult,
      provenance,
      failedPages: resp.failed_pages,
    });
  }

  return processorResult;
}