Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { config } from "../../../../../config";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import type { PDFProcessorResult } from "../types";
import { safeMarkdownToHtml } from "../markdownToHtml";
import { scrapePDFWithFirePDF } from "../firePDF";
import { cancelJob } from "./cancel";
import { maybeSaveResult, provenanceFromResponse, tryGetCached } from "./cache";
import { resolvePdfCacheKey } from "../../../../../lib/gcs-pdf-cache";
import { isUsOnly, US_DATA_REGION } from "../../../../../lib/us-only";
import { isReservedCapacity } from "../../../../../lib/reserved-capacity";
import {
firePdfReservedCapacityRefusal,
firePdfUsOnlyRefusal,
} from "./data-region";
import {
firePdfAsyncAbandonedTotal,
firePdfAsyncTotalDurationSeconds,
type AbandonedPhase,
} from "./metrics";
import { pollUntilTerminal } from "./poll";
import { fetchResult } from "./result";
import type { FirePdfByReferenceInput } from "./by-reference";
import type { FirePdfAdoptedJobInput } from "./lookup";
import { FIRE_PDF_ASYNC_MIN_REMAINING_MS } from "./routing";
import { POLL_TIMEOUT_BUFFER_MS } from "./schema";
import { submitJob, SubmitJobMayHaveBeenAcceptedError } from "./submit";
import {
computeByReferenceDeadlineMs,
computeDeadlineMs,
computeInlineJobDeadlineMs,
defaultSleep,
failAsync,
FirePdfAsyncFailure,
} from "./utils";
export { FirePdfAsyncFailure };
type FirePdfAsyncDeps = {
fetchImpl?: typeof undiciFetch;
fallbackImpl?: typeof scrapePDFWithFirePDF;
sleepImpl?: (ms: number, signal?: AbortSignal) => Promise<void>;
nowImpl?: () => number;
randomImpl?: () => number;
};
export async function scrapePDFWithFirePDFAsync(
meta: Meta,
/** Inline base64 (string, the historical shape), a pre-uploaded GCS
* reference for large files, or an existing job to adopt (found by
* content lookup β no upload, no submit, just poll + fetch). Neither
* non-string form has a sync fallback β the bytes don't fit fire-pdf's
* inline paths β so infra fallbacks below only apply to the string
* form. */
input: string | FirePdfByReferenceInput | FirePdfAdoptedJobInput,
maxPages?: number,
pagesProcessed?: number,
mode?: PDFMode,
deps: FirePdfAsyncDeps = {},
includePageMarkdown = false,
includeBlocks = false,
pageMarkers = false,
): Promise<PDFProcessorResult> {
const fetchImpl = deps.fetchImpl ?? undiciFetch;
const fallbackImpl = deps.fallbackImpl ?? scrapePDFWithFirePDF;
const sleep = deps.sleepImpl ?? defaultSleep;
const now = deps.nowImpl ?? Date.now;
const random = deps.randomImpl ?? Math.random;
const base64Content = typeof input === "string" ? input : undefined;
const adopted =
typeof input !== "string" && "adoptScrapeId" in input ? input : undefined;
const byReference =
typeof input !== "string" && "gcsUri" in input ? input : undefined;
// Which fire-pdf job this attempt watches: our own scrape_id, or the
// adopted job's. Poll, result-fetch, and cancel must all agree.
const jobScrapeId = adopted ? adopted.adoptScrapeId : meta.id;
// Async persists inputs and queue state, so ZDR is excluded until that
// lifecycle has an explicit delete-on-completion contract.
if (meta.internalOptions.zeroDataRetention) {
if (base64Content === undefined) {
// By-reference already persisted the input object, and adoption
// reads another submitter's persisted job; the routing layer must
// never send ZDR traffic here.
throw new Error(
"fire-pdf by-reference submit is not available under zero data retention",
);
}
return fallbackImpl(
meta,
base64Content,
maxPages,
pagesProcessed,
mode,
includePageMarkdown,
includeBlocks,
pageMarkers,
);
}
// Cache addressing: inline submits keep the historical key (sha256 of
// the base64 payload); by-reference submits use a `raw-` prefixed key
// over the raw-byte sha, so repeat scrapes of the same large document
// don't reprocess it. On the direct-bucket path the two keyspaces are
// distinct β an inline and a by-reference parse of the same document do
// not share entries; the cache service is asked with both keys. The
// LOOKUP for by-reference happens at the call site BEFORE
// the input object is uploaded (a hit must skip the 30-256MB transfer,
// which has already happened by the time this function runs); only the
// inline path looks up here. Both paths save here.
const cacheInput =
typeof input === "string"
? input
: { key: `raw-${input.sha256.toLowerCase()}` };
const cached =
typeof input === "string"
? await tryGetCached(
meta,
cacheInput,
mode,
maxPages,
pagesProcessed,
includePageMarkdown,
includeBlocks,
pageMarkers,
)
: null;
if (cached) return cached;
meta.abort.throwIfAborted();
const remainingMs = meta.abort.scrapeTimeout();
if (
remainingMs !== undefined &&
remainingMs < FIRE_PDF_ASYNC_MIN_REMAINING_MS
) {
failAsync(meta, "deadline_too_close", { remainingMs });
}
const baseUrl = config.FIRE_PDF_BASE_URL;
if (!baseUrl) {
// Should be unreachable β call site checks this β but fall back rather
// than crash if a route somehow bypasses the gate.
if (base64Content === undefined) {
throw new Error(
"fire-pdf by-reference submit requires FIRE_PDF_BASE_URL",
);
}
return fallbackImpl(
meta,
base64Content,
maxPages,
pagesProcessed,
mode,
includePageMarkdown,
includeBlocks,
pageMarkers,
);
}
const overallStartedAt = now();
const submitTime = now();
// The caller's window governs how long THIS attempt polls: scrapeURLLoop
// kills no-timeout scrapes at 5 minutes regardless of what the job is
// allowed, so polling past it only burns a dead scrape's cycles.
const callerWindowMs = computeDeadlineMs(remainingMs);
// The JOB deadline is decoupled from the caller for by-reference
// submits: page-scaled, never below the caller window, capped at
// 30 min. An inline job's deadline sits INSIDE the caller window by a
// margin (computeInlineJobDeadlineMs): when its caller dies the job is
// cancelled and the work discarded, so the worker must reach its own
// deadline β and write the deadline-degraded result it produces there β
// while the caller is still around to poll and fetch it. A by-reference
// job instead outlives its caller on purpose (cancel is skipped below):
// it finishes server-side, lands in the raw-sha cache and the
// content-adoption lookup, and the customer's retry β with a fresh
// scrape_id β converges instead of restarting a multi-minute document
// from zero. Callers wanting first-attempt success must still pass an
// explicit `timeout`.
const deadlineFromNow = byReference
? computeByReferenceDeadlineMs(remainingMs, pagesProcessed)
: computeInlineJobDeadlineMs(callerWindowMs);
const jobDeadlineAtMs = submitTime + deadlineFromNow;
const deadlineAt = new Date(jobDeadlineAtMs).toISOString();
const pollingDeadline = submitTime + callerWindowMs + POLL_TIMEOUT_BUFFER_MS;
// Account context for FirePDF's per-team admission observation,
// snapshotted from the request ACUC into internalOptions at
// acceptance (same pattern as teamFlags) β no re-fetch here. Absence
// means FirePDF skips team observation for this submit.
const rawConcurrency = meta.internalOptions.teamConcurrency;
const teamConcurrency =
typeof rawConcurrency === "number" && rawConcurrency > 0
? rawConcurrency
: undefined;
// ββ Step 1: POST /jobs (skipped when adopting an existing job) ββββββββ
let submissionAccepted = false;
let terminalReached = false;
let lastNonTerminalStatus: "queued" | "published" | "running" | undefined;
let polled: Awaited<ReturnType<typeof pollUntilTerminal>>;
let fetched: Awaited<ReturnType<typeof fetchResult>>;
// What goes on the wire for step 1 β null means nothing does: an
// adopted job was submitted by an earlier attempt and is only watched.
const wireInput = byReference
? ({
kind: "byReference",
gcsUri: byReference.gcsUri,
sha256: byReference.sha256,
} as const)
: base64Content !== undefined
? ({ kind: "inline", base64Content } as const)
: null;
// Ownership policy, in one place: only an abandoned INLINE job is
// cancelled. An adopted job is not ours to kill β its owner or other
// retries may still be watching. A by-reference job is left running BY
// DESIGN: its input is durably content-addressed, so the completion
// this caller never sees still lands in the raw-sha cache and the
// adoption lookup, converting the customer's retry loop into a cache
// hit. The cost is bounded β the job's own deadline (β€30 min) β and
// strictly smaller than the redo loop it replaces.
const cancelOnAbandon = wireInput?.kind === "inline";
try {
let alreadyDone = false;
let initialDelay: number | undefined;
if (wireInput === null) {
meta.logger.info("FirePDF async adopting existing job", {
scrapeId: meta.id,
adoptedScrapeId: jobScrapeId,
});
// Adopted jobs are live by definition (done ones return on the
// first poll); the true start time is unknown, so the estimate
// conservatively counts from now. Written into the shared
// container so it survives the spread copies between here and the
// outer timeout handler.
if (meta.largePdfProcessing) {
meta.largePdfProcessing.current = {
jobScrapeId,
pagesEstimate: pagesProcessed,
submittedAtMs: submitTime,
lastStatus: "running",
};
}
} else {
const submit = await submitJob({
meta,
baseUrl,
input: wireInput,
maxPages,
pagesProcessed,
mode,
includePageMarkdown,
includeBlocks,
pageMarkers,
deadlineAt,
teamConcurrency,
fetchImpl,
sleep,
});
submissionAccepted = true;
alreadyDone = submit.alreadyDone;
initialDelay = submit.retryAfterMs;
if (byReference && !alreadyDone && meta.largePdfProcessing) {
// The job now exists server-side and (per the cancel policy
// above) will keep running if this scrape is abandoned β record
// enough state for the timeout error to say so. Written into the
// shared container so it survives the spread copies between here
// and the outer timeout handler.
meta.largePdfProcessing.current = {
jobScrapeId,
pagesEstimate: pagesProcessed,
submittedAtMs: submitTime,
jobDeadlineAtMs,
lastStatus: "queued",
};
}
}
terminalReached = alreadyDone;
// ββ Step 2: poll until terminal (skip on idempotent-replay done) ββββββ
polled = alreadyDone
? {
poll: { scrape_id: jobScrapeId, status: "done" as const },
pollCount: 0,
}
: await pollUntilTerminal({
baseUrl,
scrapeId: jobScrapeId,
initialDelay,
pagesEstimate: pagesProcessed,
longPollWaitMs: config.FIRE_PDF_ASYNC_WAIT_MS,
pollingDeadline,
meta,
fetchImpl,
sleep,
now,
random,
// Only a job WE submitted inline has a deadline we know and that
// sits inside this caller's window. An adopted job's deadline is
// its owner's; a by-reference job's lies beyond the window.
jobDeadlineAtMs:
wireInput?.kind === "inline" ? jobDeadlineAtMs : undefined,
onNonTerminalStatus: (status, estimatedRemainingMs) => {
lastNonTerminalStatus = status;
const current = meta.largePdfProcessing?.current;
if (!current) return;
current.lastStatus = status;
if (estimatedRemainingMs !== undefined) {
current.serverEstimate = {
remainingMs: estimatedRemainingMs,
observedAtMs: now(),
};
}
},
});
terminalReached = true;
// ββ Step 3: GET /jobs/:id/result ββββββββββββββββββββββββββββββββββββ
fetched = await fetchResult({
baseUrl,
scrapeId: jobScrapeId,
meta,
fetchImpl,
sleep,
});
} catch (error) {
const submitMayHaveBeenAccepted =
error instanceof SubmitJobMayHaveBeenAcceptedError;
const jobAlreadyTerminal =
error instanceof FirePdfAsyncFailure &&
(error.reason === "terminal_failed" ||
error.reason === "terminal_expired" ||
error.reason === "terminal_cancelled" ||
error.reason === "region_unavailable" ||
error.reason === "reserved_unavailable");
if ((jobAlreadyTerminal || terminalReached) && meta.largePdfProcessing) {
// The job is dead β a "processing continues" message would lie.
meta.largePdfProcessing.current = undefined;
}
const abortError =
error instanceof AbortManagerThrownError
? error
: submitMayHaveBeenAccepted &&
error.originalError instanceof AbortManagerThrownError
? error.originalError
: undefined;
if (abortError?.tier === "scrape") {
// The caller's scrape window closed first β the outcome the job
// deadline margin exists to avoid β so count where it happened.
// Engine-tier aborts (the waterfall moving on) and external aborts
// (the client hung up) are not that signal.
const phase: AbandonedPhase =
wireInput !== null && !submissionAccepted
? "submit"
: terminalReached
? "result"
: "poll";
firePdfAsyncAbandonedTotal.labels(phase).inc();
meta.logger.warn("FirePDF async abandoned by caller abort", {
scrapeId: meta.id,
event: "fire_pdf_async_abandoned",
phase,
tier: abortError.tier,
lastStatus: lastNonTerminalStatus,
// Only meaningful for a job this attempt submitted itself.
jobDeadlineInMs:
wireInput !== null ? jobDeadlineAtMs - now() : undefined,
elapsedMs: now() - overallStartedAt,
});
}
if (
cancelOnAbandon &&
(submissionAccepted || submitMayHaveBeenAccepted) &&
!terminalReached &&
!jobAlreadyTerminal
) {
await cancelJob({ baseUrl, scrapeId: meta.id, meta, fetchImpl });
}
if (
error instanceof FirePdfAsyncFailure &&
error.reason === "region_unavailable"
) {
throw firePdfUsOnlyRefusal(meta, "region_unavailable");
}
if (
error instanceof FirePdfAsyncFailure &&
error.reason === "reserved_unavailable"
) {
throw firePdfReservedCapacityRefusal(meta, "reserved_unavailable");
}
throw submitMayHaveBeenAccepted ? error.originalError : error;
}
// The job reached a terminal state and its result was fetched β nothing
// "continues", regardless of how the validations below turn out (their
// failAsync throws must not leave stale processing state behind).
if (meta.largePdfProcessing) {
meta.largePdfProcessing.current = undefined;
}
// Adopted jobs included: a job that ran before the pin existed carries no
// echo, and its result may have come from GPU capacity outside the US.
if (
isUsOnly(meta.internalOptions.teamFlags ?? undefined) &&
fetched.data_region_applied !== US_DATA_REGION
) {
throw firePdfUsOnlyRefusal(meta, "region_unconfirmed");
}
if (
isReservedCapacity(meta.internalOptions.teamFlags ?? undefined) &&
fetched.reserved_applied !== true
) {
throw firePdfReservedCapacityRefusal(meta, "reserved_unconfirmed");
}
// ββ Assemble + cache save βββββββββββββββββββββββββββββββββββββββββββββ
const pages =
fetched.pages_processed ?? polled.poll.pages_processed ?? pagesProcessed;
if (includePageMarkdown && fetched.pages === undefined) {
failAsync(meta, "http_5xx", {
note: "FirePDF result omitted requested physical page markdown",
});
}
if (includeBlocks && fetched.blocks === undefined) {
failAsync(meta, "http_5xx", {
note: "FirePDF result omitted requested typed blocks",
});
}
if (pageMarkers && fetched.page_markers !== true) {
// Markers are baked into the markdown, so the missing echo is the only
// signal the worker build ignored the option; accepting the result
// would cache unmarked markdown under a marker cache variant. Fail the
// async attempt β the caller retries synchronously, where the same
// echo contract applies.
failAsync(meta, "http_5xx", {
note: "FirePDF result did not acknowledge requested page markers",
});
}
const durationMs = now() - overallStartedAt;
firePdfAsyncTotalDurationSeconds
.labels(meta.internalOptions.crawlId ? "bulk" : "interactive")
.observe(durationMs / 1000);
const cacheKey = resolvePdfCacheKey(cacheInput);
const provenance = provenanceFromResponse(fetched.provenance, meta.logger, {
scrapeId: meta.id,
cacheKey,
});
meta.logger.info("FirePDF async completed", {
scrapeId: meta.id,
durationMs,
markdownLength: fetched.markdown.length,
pagesProcessed: pages,
pageMarkdownPages: fetched.pages?.length,
blockPages: fetched.blocks?.length,
failedPages: fetched.failed_pages,
partialPages: fetched.partial_pages,
pollCount: polled.pollCount,
// The content-cache key and the producer, so a report can be turned
// into keys to purge and a result can be tied to a fire-pdf build.
cacheKey,
generation: provenance?.generation ?? "unknown",
buildSha: provenance?.build_sha ?? "unknown",
});
const processorResult: PDFProcessorResult & { markdown: string } = {
markdown: fetched.markdown,
html: await safeMarkdownToHtml(fetched.markdown, meta.logger, meta.id),
pagesProcessed: pages,
...(fetched.pages ? { pageMarkdown: fetched.pages } : {}),
...(fetched.blocks ? { blocks: fetched.blocks } : {}),
};
await maybeSaveResult({
meta,
base64Content: cacheInput,
mode,
maxPages,
includePageMarkdown,
includeBlocks,
pageMarkers,
result: processorResult,
provenance,
failedPages: fetched.failed_pages,
});
return processorResult;
}