Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import type { FirePdfTrainingCapture } from "./request-metadata";
import { config } from "../../../../../config";
import {
INLINE_JOB_DEADLINE_MARGIN_FRACTION,
INLINE_JOB_DEADLINE_MARGIN_MAX_MS,
INLINE_JOB_DEADLINE_MARGIN_MIN_MS,
JOB_DEADLINE_POLL_GRACE_MS,
MAX_DEADLINE_MS,
MIN_DEADLINE_MS,
POLL_FLOOR_MS,
POLL_CAP_MS,
} from "./schema";
import { firePdfAsyncFallbackTotal, type FallbackReason } from "./metrics";
export function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
return new Promise((resolve, reject) => {
const handle = setTimeout(() => {
signal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
const onAbort = () => {
clearTimeout(handle);
reject(
signal?.reason instanceof Error ? signal.reason : new Error("Aborted"),
);
};
if (signal) {
if (signal.aborted) {
clearTimeout(handle);
reject(
signal.reason instanceof Error ? signal.reason : new Error("Aborted"),
);
return;
}
signal.addEventListener("abort", onAbort, { once: true });
}
});
}
export function nextPollDelay(
prev: number,
retryAfterMs: number | undefined,
random: () => number = Math.random,
): number {
const candidate = Math.max(prev * 2, retryAfterMs ?? 0, POLL_FLOOR_MS);
const jittered = Math.round(candidate * (1 + random() * 0.2));
return Math.min(POLL_CAP_MS, jittered);
}
/**
* Deadline for an INLINE job: the caller window minus a margin, never below
* fire-pdf's 5s minimum. The worker plans its work against this deadline and
* degrades what is left when it arrives; the margin is what lets that result
* be written, observed by a poll, and fetched before the caller's own abort
* fires at the end of the window. Advertising the full window instead means
* the worker is still finishing when the caller gives up β the job is
* cancelled and the caller sees a timeout with nothing to show for it.
*/
export function computeInlineJobDeadlineMs(callerWindowMs: number): number {
const margin = Math.min(
INLINE_JOB_DEADLINE_MARGIN_MAX_MS,
Math.max(
INLINE_JOB_DEADLINE_MARGIN_MIN_MS,
Math.round(callerWindowMs * INLINE_JOB_DEADLINE_MARGIN_FRACTION),
),
);
return Math.max(MIN_DEADLINE_MS, callerWindowMs - margin);
}
/**
* Cap a poll delay so a poll lands just after the job deadline, and keep
* polling at the floor once the deadline has passed. Backoff (up to
* POLL_CAP_MS plus jitter) is right while the job is mid-flight; around the
* deadline it is exactly wrong, because the result appears there and the
* caller's window closes shortly after. A job deadline beyond the caller
* window (by-reference) leaves the delay untouched.
*/
export function alignPollDelay(
delayMs: number,
nowMs: number,
jobDeadlineAtMs: number | undefined,
): number {
if (jobDeadlineAtMs === undefined) return delayMs;
const untilTarget = jobDeadlineAtMs + JOB_DEADLINE_POLL_GRACE_MS - nowMs;
if (untilTarget <= 0) return Math.min(delayMs, POLL_FLOOR_MS);
return Math.min(delayMs, Math.max(POLL_FLOOR_MS, untilTarget));
}
export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {
// 5min default when there's no scrape budget. This matches the usable
// window exactly: scrapeURLLoop races every engine against
// `scrapeTimeout() ?? 300000`, so a no-timeout scrape dies at 5 minutes
// regardless of what we advertise to FirePDF. Long documents (large
// by-reference PDFs especially) need an explicit caller `timeout`, which
// unlocks up to MAX_DEADLINE_MS here and extends the loop's race alike.
// Only cap the upper bound so we never advertise more time to FirePDF
// than the caller actually has.
const fallback = 5 * 60 * 1_000;
const candidate = scrapeTimeoutMs ?? fallback;
return Math.min(MAX_DEADLINE_MS, candidate);
}
/** Worst-case processing rate for the page-scaled by-reference deadline.
* Must cover the SLOW class, not the median: scanned giants run full-page
* OCR at ~1.3s/page p90 in prod (2026-08-27), and the original 500ms β
* calibrated on a text-heavy 6,543-page run β starved them into deadline
* fallback (a 798-page scan got 11.6min against ~17min of real OCR).
*
* The base must absorb QUEUE WAIT, which spends the same budget as
* processing (the deadline clock starts at submit, not claim): measured
* bursts queued giants 12-14 minutes behind five xl workers, so a 5-min
* base guaranteed starvation regardless of the per-page term (a 931-page
* doc reached its worker with 84s left and degraded 93% of its pages β
* and that degraded result then poisoned content-adoption for every
* retry). 10 min covers observed burst queues; genuinely fixing the
* queue/processing conflation (claim-time budgets) is fire-pdf-side
* follow-up work.
*/
const BY_REFERENCE_DEADLINE_PER_PAGE_MS = 1_250;
const BY_REFERENCE_DEADLINE_BASE_MS = 10 * 60 * 1_000;
/**
* Job deadline for by-reference submits, DECOUPLED from the caller's
* remaining budget: what the document needs (page-scaled), never less
* than the caller's own window, capped at fire-pdf's 30-min maximum.
*
* Inline submits advertise exactly the caller window (computeDeadlineMs)
* because an abandoned inline job is cancelled and its work discarded.
* A by-reference job that outlives its caller is NOT discarded: the
* async caller skips cancel-on-abandon, the job finishes server-side,
* and the result lands in the raw-sha cache and the content-adoption
* lookup β so the customer's retry converges instead of restarting a
* multi-minute document from zero on every attempt.
*/
export function computeByReferenceDeadlineMs(
scrapeTimeoutMs: number | undefined,
pagesEstimate: number | undefined,
): number {
const callerWindow = computeDeadlineMs(scrapeTimeoutMs);
const pages =
pagesEstimate !== undefined && pagesEstimate > 0 ? pagesEstimate : 0;
const pageScaled =
BY_REFERENCE_DEADLINE_BASE_MS + pages * BY_REFERENCE_DEADLINE_PER_PAGE_MS;
return Math.min(MAX_DEADLINE_MS, Math.max(callerWindow, pageScaled));
}
/**
* The `options` object for fire-pdf async wire calls. One builder for
* both POST /jobs and POST /jobs/lookup: adoption matches on fire-pdf's
* idempotency options, so a lookup that built its options differently
* from the submit would silently never match.
*/
export function buildFirePdfJobOptions(args: {
maxPages: number | undefined;
pagesProcessed: number | undefined;
mode: PDFMode | undefined;
includePageMarkdown: boolean;
includeBlocks: boolean;
pageMarkers: boolean;
tableFormat: string | undefined;
trainingCapture: FirePdfTrainingCapture | undefined;
dataRegion: string | undefined;
reserved?: boolean;
}): Record<string, unknown> {
return {
// fire-pdf accepts only a positive integer here. When the router has no
// page count (0), the field is left out and fire-pdf counts the pages
// itself on inline submits. By-reference submits require a positive
// count and check for one before they get here.
...(args.pagesProcessed !== undefined &&
Number.isInteger(args.pagesProcessed) &&
args.pagesProcessed > 0 && {
pages_estimate: args.pagesProcessed,
}),
...(args.maxPages !== undefined && { max_pages: args.maxPages }),
...(args.mode !== undefined && { mode: args.mode }),
...(args.includePageMarkdown && { include_page_markdown: true }),
...(args.includeBlocks && { include_blocks: true }),
// Intentionally camelCase, unlike its siblings: the fire-pdf async
// /jobs options schema named this key `pageMarkers` (fire-pdf
// api/src/http/schemas/jobs.ts) while the sync /ocr path uses
// `page_markers`. Sending snake_case here would be rejected as an
// unknown option.
...(args.pageMarkers && { pageMarkers: true }),
...(args.tableFormat !== undefined && { tableFormat: args.tableFormat }),
...(args.trainingCapture && { training_capture: args.trainingCapture }),
// An idempotency key on the PDF service, so a pinned lookup only adopts
// a pinned job.
...(args.dataRegion && { data_region: args.dataRegion }),
// Also an idempotency key, so a restricted lookup only adopts a job
// restricted the same way.
...(args.reserved && { reserved: true }),
};
}
export function firePdfHeaders(includeJson = false): Record<string, string> {
return {
...(includeJson && { "Content-Type": "application/json" }),
...(config.FIRE_PDF_API_KEY && {
Authorization: `Bearer ${config.FIRE_PDF_API_KEY}`,
}),
};
}
export class FirePdfAsyncFailure extends Error {
constructor(
public readonly reason: FallbackReason,
public readonly extra: Record<string, unknown> = {},
) {
super(`fire-pdf async failed: ${reason}`);
this.name = "FirePdfAsyncFailure";
}
}
export function failAsync(
meta: Meta,
reason: FallbackReason,
extra: Record<string, unknown> = {},
): never {
firePdfAsyncFallbackTotal.labels(reason).inc();
meta.logger.warn("FirePDF async failed", {
scrapeId: meta.id,
reason,
...extra,
});
throw new FirePdfAsyncFailure(reason, extra);
}