Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

9.1 KB
Raw Blame
import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import type { FirePdfTrainingCapture } from "./request-metadata";
import { config } from "../../../../../config";
import {
  INLINE_JOB_DEADLINE_MARGIN_FRACTION,
  INLINE_JOB_DEADLINE_MARGIN_MAX_MS,
  INLINE_JOB_DEADLINE_MARGIN_MIN_MS,
  JOB_DEADLINE_POLL_GRACE_MS,
  MAX_DEADLINE_MS,
  MIN_DEADLINE_MS,
  POLL_FLOOR_MS,
  POLL_CAP_MS,
} from "./schema";
import { firePdfAsyncFallbackTotal, type FallbackReason } from "./metrics";

export function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
  return new Promise((resolve, reject) => {
    const handle = setTimeout(() => {
      signal?.removeEventListener("abort", onAbort);
      resolve();
    }, ms);
    const onAbort = () => {
      clearTimeout(handle);
      reject(
        signal?.reason instanceof Error ? signal.reason : new Error("Aborted"),
      );
    };
    if (signal) {
      if (signal.aborted) {
        clearTimeout(handle);
        reject(
          signal.reason instanceof Error ? signal.reason : new Error("Aborted"),
        );
        return;
      }
      signal.addEventListener("abort", onAbort, { once: true });
    }
  });
}

export function nextPollDelay(
  prev: number,
  retryAfterMs: number | undefined,
  random: () => number = Math.random,
): number {
  const candidate = Math.max(prev * 2, retryAfterMs ?? 0, POLL_FLOOR_MS);
  const jittered = Math.round(candidate * (1 + random() * 0.2));
  return Math.min(POLL_CAP_MS, jittered);
}

/**
 * Deadline for an INLINE job: the caller window minus a margin, never below
 * fire-pdf's 5s minimum. The worker plans its work against this deadline and
 * degrades what is left when it arrives; the margin is what lets that result
 * be written, observed by a poll, and fetched before the caller's own abort
 * fires at the end of the window. Advertising the full window instead means
 * the worker is still finishing when the caller gives up β€” the job is
 * cancelled and the caller sees a timeout with nothing to show for it.
 */
export function computeInlineJobDeadlineMs(callerWindowMs: number): number {
  const margin = Math.min(
    INLINE_JOB_DEADLINE_MARGIN_MAX_MS,
    Math.max(
      INLINE_JOB_DEADLINE_MARGIN_MIN_MS,
      Math.round(callerWindowMs * INLINE_JOB_DEADLINE_MARGIN_FRACTION),
    ),
  );
  return Math.max(MIN_DEADLINE_MS, callerWindowMs - margin);
}

/**
 * Cap a poll delay so a poll lands just after the job deadline, and keep
 * polling at the floor once the deadline has passed. Backoff (up to
 * POLL_CAP_MS plus jitter) is right while the job is mid-flight; around the
 * deadline it is exactly wrong, because the result appears there and the
 * caller's window closes shortly after. A job deadline beyond the caller
 * window (by-reference) leaves the delay untouched.
 */
export function alignPollDelay(
  delayMs: number,
  nowMs: number,
  jobDeadlineAtMs: number | undefined,
): number {
  if (jobDeadlineAtMs === undefined) return delayMs;
  const untilTarget = jobDeadlineAtMs + JOB_DEADLINE_POLL_GRACE_MS - nowMs;
  if (untilTarget <= 0) return Math.min(delayMs, POLL_FLOOR_MS);
  return Math.min(delayMs, Math.max(POLL_FLOOR_MS, untilTarget));
}

export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {
  // 5min default when there's no scrape budget. This matches the usable
  // window exactly: scrapeURLLoop races every engine against
  // `scrapeTimeout() ?? 300000`, so a no-timeout scrape dies at 5 minutes
  // regardless of what we advertise to FirePDF. Long documents (large
  // by-reference PDFs especially) need an explicit caller `timeout`, which
  // unlocks up to MAX_DEADLINE_MS here and extends the loop's race alike.
  // Only cap the upper bound so we never advertise more time to FirePDF
  // than the caller actually has.
  const fallback = 5 * 60 * 1_000;
  const candidate = scrapeTimeoutMs ?? fallback;
  return Math.min(MAX_DEADLINE_MS, candidate);
}

/** Worst-case processing rate for the page-scaled by-reference deadline.
 * Must cover the SLOW class, not the median: scanned giants run full-page
 * OCR at ~1.3s/page p90 in prod (2026-08-27), and the original 500ms β€”
 * calibrated on a text-heavy 6,543-page run β€” starved them into deadline
 * fallback (a 798-page scan got 11.6min against ~17min of real OCR).
 *
 * The base must absorb QUEUE WAIT, which spends the same budget as
 * processing (the deadline clock starts at submit, not claim): measured
 * bursts queued giants 12-14 minutes behind five xl workers, so a 5-min
 * base guaranteed starvation regardless of the per-page term (a 931-page
 * doc reached its worker with 84s left and degraded 93% of its pages β€”
 * and that degraded result then poisoned content-adoption for every
 * retry). 10 min covers observed burst queues; genuinely fixing the
 * queue/processing conflation (claim-time budgets) is fire-pdf-side
 * follow-up work.
 */
const BY_REFERENCE_DEADLINE_PER_PAGE_MS = 1_250;
const BY_REFERENCE_DEADLINE_BASE_MS = 10 * 60 * 1_000;

/**
 * Job deadline for by-reference submits, DECOUPLED from the caller's
 * remaining budget: what the document needs (page-scaled), never less
 * than the caller's own window, capped at fire-pdf's 30-min maximum.
 *
 * Inline submits advertise exactly the caller window (computeDeadlineMs)
 * because an abandoned inline job is cancelled and its work discarded.
 * A by-reference job that outlives its caller is NOT discarded: the
 * async caller skips cancel-on-abandon, the job finishes server-side,
 * and the result lands in the raw-sha cache and the content-adoption
 * lookup β€” so the customer's retry converges instead of restarting a
 * multi-minute document from zero on every attempt.
 */
export function computeByReferenceDeadlineMs(
  scrapeTimeoutMs: number | undefined,
  pagesEstimate: number | undefined,
): number {
  const callerWindow = computeDeadlineMs(scrapeTimeoutMs);
  const pages =
    pagesEstimate !== undefined && pagesEstimate > 0 ? pagesEstimate : 0;
  const pageScaled =
    BY_REFERENCE_DEADLINE_BASE_MS + pages * BY_REFERENCE_DEADLINE_PER_PAGE_MS;
  return Math.min(MAX_DEADLINE_MS, Math.max(callerWindow, pageScaled));
}

/**
 * The `options` object for fire-pdf async wire calls. One builder for
 * both POST /jobs and POST /jobs/lookup: adoption matches on fire-pdf's
 * idempotency options, so a lookup that built its options differently
 * from the submit would silently never match.
 */
export function buildFirePdfJobOptions(args: {
  maxPages: number | undefined;
  pagesProcessed: number | undefined;
  mode: PDFMode | undefined;
  includePageMarkdown: boolean;
  includeBlocks: boolean;
  pageMarkers: boolean;
  tableFormat: string | undefined;
  trainingCapture: FirePdfTrainingCapture | undefined;
  dataRegion: string | undefined;
  reserved?: boolean;
}): Record<string, unknown> {
  return {
    // fire-pdf accepts only a positive integer here. When the router has no
    // page count (0), the field is left out and fire-pdf counts the pages
    // itself on inline submits. By-reference submits require a positive
    // count and check for one before they get here.
    ...(args.pagesProcessed !== undefined &&
      Number.isInteger(args.pagesProcessed) &&
      args.pagesProcessed > 0 && {
        pages_estimate: args.pagesProcessed,
      }),
    ...(args.maxPages !== undefined && { max_pages: args.maxPages }),
    ...(args.mode !== undefined && { mode: args.mode }),
    ...(args.includePageMarkdown && { include_page_markdown: true }),
    ...(args.includeBlocks && { include_blocks: true }),
    // Intentionally camelCase, unlike its siblings: the fire-pdf async
    // /jobs options schema named this key `pageMarkers` (fire-pdf
    // api/src/http/schemas/jobs.ts) while the sync /ocr path uses
    // `page_markers`. Sending snake_case here would be rejected as an
    // unknown option.
    ...(args.pageMarkers && { pageMarkers: true }),
    ...(args.tableFormat !== undefined && { tableFormat: args.tableFormat }),
    ...(args.trainingCapture && { training_capture: args.trainingCapture }),
    // An idempotency key on the PDF service, so a pinned lookup only adopts
    // a pinned job.
    ...(args.dataRegion && { data_region: args.dataRegion }),
    // Also an idempotency key, so a restricted lookup only adopts a job
    // restricted the same way.
    ...(args.reserved && { reserved: true }),
  };
}

export function firePdfHeaders(includeJson = false): Record<string, string> {
  return {
    ...(includeJson && { "Content-Type": "application/json" }),
    ...(config.FIRE_PDF_API_KEY && {
      Authorization: `Bearer ${config.FIRE_PDF_API_KEY}`,
    }),
  };
}

export class FirePdfAsyncFailure extends Error {
  constructor(
    public readonly reason: FallbackReason,
    public readonly extra: Record<string, unknown> = {},
  ) {
    super(`fire-pdf async failed: ${reason}`);
    this.name = "FirePdfAsyncFailure";
  }
}

export function failAsync(
  meta: Meta,
  reason: FallbackReason,
  extra: Record<string, unknown> = {},
): never {
  firePdfAsyncFallbackTotal.labels(reason).inc();
  meta.logger.warn("FirePDF async failed", {
    scrapeId: meta.id,
    reason,
    ...extra,
  });
  throw new FirePdfAsyncFailure(reason, extra);
}