Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

10.0 KB
Raw Blame
import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import type { PDFProcessorResult } from "../types";
import { scrapePDFWithFirePDFAsync } from "./async";
import {
  rewritePdfInputForFirePdf,
  sha256OfFile,
  uploadPdfInputForFirePdf,
} from "./by-reference";
import { cacheKeyShape, tryGetCached } from "./cache";
import { lookupAdoptableFirePdfJob } from "./lookup";
import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";
import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";
import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
import {
  buildFirePdfJobTrainingCapture,
  firePdfTableFormat,
} from "./request-metadata";

type FirePdfByReferenceAttemptArgs = {
  meta: Meta;
  tempFilePath: string;
  fileSizeBytes: number;
  /** Detected page count; the caller guarantees it is positive (fire-pdf
   * rejects by-reference submits without one). */
  pagesEstimate: number;
  mode: PDFMode | undefined;
  maxPages: number | undefined;
  includePageMarkdown: boolean;
  includeBlocks: boolean;
  pageMarkers: boolean;
};

/**
 * The full by-reference attempt for one large PDF, in cost order β€” each
 * step only runs when the previous one didn't already produce a result:
 *
 *   1. raw-sha cache lookup  (one streamed disk read)
 *   2. content adoption      (poll a live/done job for the same
 *                             bytes+options β€” no upload, no reprocessing)
 *   3. handoff rewrite or streaming upload, then a fresh async submit
 *
 * Returns the processor result, or null when the input could not even be
 * placed in the fire-pdf bucket (rewrite AND upload failed) β€” the caller
 * falls through to the legacy chain, preserving pre-by-reference
 * behavior. Anything after a successful placement throws on failure
 * instead: there is no inline retry at this size, and the legacy chain
 * would silently degrade a large document to text-only extraction.
 *
 * Kept out of the engine router on purpose: this is transport/lifecycle
 * recovery, not routing.
 */
export async function runFirePdfByReferenceAttempt(
  args: FirePdfByReferenceAttemptArgs,
): Promise<PDFProcessorResult | null> {
  const {
    meta,
    tempFilePath,
    fileSizeBytes,
    pagesEstimate,
    mode,
    maxPages,
    includePageMarkdown,
    includeBlocks,
    pageMarkers,
  } = args;

  // Cache BEFORE upload: the raw-byte sha is the by-reference cache
  // identity, and it must be checked before the 30-256MB transfer β€”
  // a repeat scrape of the same document should cost one streamed
  // disk read, not a full re-upload. scrapePDFWithFirePDFAsync
  // deliberately skips the by-reference lookup for the same reason
  // (it runs post-upload) and only saves.
  //
  // The pre-hash is unconditional on this path: besides the cache
  // key, it is the content-adoption identity (fire-pdf's
  // POST /jobs/lookup) and it verifies a fire-engine handoff
  // before the server-side copy. Uncacheable requests (maxPages)
  // skip only the cache LOOKUP β€” they still adopt; gating the hash
  // on cacheability (the pre-adoption rule) would permanently lock
  // those requests out of adoption. The cost is one extra streamed
  // disk read β€” seconds even at 256MB β€” against the upload and
  // multi-minute processing run an adoption hit skips. A failed
  // pre-hash falls through to the upload path (which hashes
  // in-pipeline), never errors the scrape.
  const handoff = meta.pdfPrefetch?.gcsReference;
  const { cacheable: byRefCacheable } = cacheKeyShape(
    mode,
    maxPages,
    includePageMarkdown,
    includeBlocks,
    pageMarkers,
  );
  // asSignal() after an abort returns a signal whose listeners never
  // fire β€” check the manager directly (outside the swallowing catch
  // below) so an already-cancelled scrape doesn't stream-read the whole
  // file first.
  meta.abort.throwIfAborted();
  let localSha256: string | undefined;
  try {
    localSha256 = await sha256OfFile(tempFilePath, meta.abort.asSignal());
  } catch (error) {
    meta.logger.warn(
      "Pre-upload hash of large PDF failed; continuing without cache lookup",
      {
        method: "scrapePDF/firePdfByReference",
        error,
        scrape_id: meta.id,
      },
    );
  }
  const cachedByRef =
    byRefCacheable && localSha256
      ? await tryGetCached(
          meta,
          { key: `raw-${localSha256}` },
          mode,
          maxPages,
          pagesEstimate,
          includePageMarkdown,
          includeBlocks,
          pageMarkers,
        )
      : null;
  // A scrape cancelled during the hash/lookup must not return a
  // success out of the cache.
  meta.abort.throwIfAborted();
  if (cachedByRef) {
    return cachedByRef;
  }

  // Content adoption, still BEFORE upload: retries carry fresh
  // scrape_ids, so fire-pdf's scrape_id idempotency can't join
  // them to a job an earlier attempt started (and, by design,
  // never cancelled β€” see the async module's cancel policy). If a
  // job for these exact bytes+options is live or done, poll IT:
  // no upload, no duplicate processing. On success the result is
  // saved under the raw-sha cache key, so the attempt after next
  // is a pure cache hit. If the adopted job dies (expired/failed)
  // or fire-pdf errors, fall through to a fresh upload+submit β€”
  // except when the failure says this caller's own budget is
  // gone, where a fresh submit could not succeed either.
  if (localSha256) {
    const adoptable = await lookupAdoptableFirePdfJob(
      meta,
      localSha256,
      buildFirePdfJobOptions({
        maxPages,
        pagesProcessed: pagesEstimate,
        mode,
        includePageMarkdown,
        includeBlocks,
        pageMarkers,
        tableFormat: firePdfTableFormat(meta),
        trainingCapture: buildFirePdfJobTrainingCapture(meta),
        dataRegion: isUsOnly(meta.internalOptions.teamFlags ?? undefined)
          ? US_DATA_REGION
          : undefined,
        reserved: isReservedCapacity(
          meta.internalOptions.teamFlags ?? undefined,
        ),
      }),
    );
    meta.abort.throwIfAborted();
    if (adoptable) {
      try {
        return await scrapePDFWithFirePDFAsync(
          {
            ...meta,
            logger: meta.logger.child({
              method: "scrapePDF/firePDFAsyncAdopted",
            }),
          },
          adoptable,
          maxPages,
          pagesEstimate,
          mode,
          undefined,
          includePageMarkdown,
          includeBlocks,
          pageMarkers,
        );
      } catch (error) {
        // An aborted scrape must not fall through to a fresh
        // upload+submit (covers every abort shape without
        // matching error classes).
        meta.abort.throwIfAborted();
        if (
          error instanceof FirePdfAsyncFailure &&
          (error.reason === "polling_timeout" ||
            error.reason === "deadline_too_close")
        ) {
          throw error;
        }
        meta.logger.warn(
          "Adopted FirePDF job did not deliver; submitting fresh",
          {
            method: "scrapePDF/firePDFAsyncAdopted",
            event: "fire_pdf_adoption_fallthrough",
            error,
            adopted_scrape_id: adoptable.adoptScrapeId,
            scrape_id: meta.id,
          },
        );
      }
    }
  }

  // When fire-engine already handed the file off via GCS, a server-side
  // rewrite moves it into the fire-pdf input bucket without the bytes
  // transiting this process; otherwise (or if the rewrite fails)
  // stream-upload the local temp file. The handoff hash becomes
  // fire-pdf's idempotency identity, so it must match the raw-byte sha
  // already computed for the cache check above (no second disk read
  // needed); any mismatch falls back to the hashing upload.
  const handoffShaMatches =
    localSha256 !== undefined &&
    handoff?.sha256 !== undefined &&
    handoff.sha256.toLowerCase() === localSha256;
  if (
    localSha256 !== undefined &&
    handoff?.sha256 !== undefined &&
    !handoffShaMatches
  ) {
    meta.logger.warn(
      "fire-engine handoff sha256 does not match local bytes; using streaming upload",
      {
        method: "scrapePDF/firePdfByReference",
        event: "fire_pdf_handoff_sha_mismatch",
        scrape_id: meta.id,
      },
    );
  }
  const rewriteEligible =
    handoffShaMatches &&
    handoff !== undefined &&
    handoff.sizeBytes === fileSizeBytes;
  const uploaded =
    (rewriteEligible && handoff
      ? await rewritePdfInputForFirePdf(meta, {
          uri: handoff.uri,
          // rewriteEligible implies handoffShaMatches implies defined
          sha256: localSha256!,
          sizeBytes: fileSizeBytes,
          generation: handoff.generation,
        })
      : null) ??
    // A distinct key when a rewrite was attempted: a timed-out
    // copy may still complete and must never overwrite this
    // upload.
    (await uploadPdfInputForFirePdf(meta, tempFilePath, fileSizeBytes, {
      keyVariant: rewriteEligible ? "s" : undefined,
      precomputedSha256: localSha256,
    }));
  if (!uploaded) {
    // Upload failure: the caller falls through to the legacy chain β€” its
    // oversized-skip warning still fires, preserving the
    // pre-by-reference behavior.
    return null;
  }

  try {
    return await scrapePDFWithFirePDFAsync(
      {
        ...meta,
        logger: meta.logger.child({
          method: "scrapePDF/firePDFAsyncByReference",
        }),
      },
      uploaded,
      maxPages,
      pagesEstimate,
      mode,
      undefined,
      includePageMarkdown,
      includeBlocks,
      pageMarkers,
    );
  } catch (error) {
    // No inline retry exists at this size, and the legacy chain
    // below would silently degrade a large document to text-only
    // extraction. Surface the failure instead.
    meta.logger.error(
      "FirePDF by-reference scrape failed (no fallback at this size)",
      {
        method: "scrapePDF/firePDFAsyncByReference",
        error,
        file_size_bytes: fileSizeBytes,
        scrape_id: meta.id,
        team_id: meta.internalOptions.teamId,
      },
    );
    throw error;
  }
}