Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { z } from "zod";
// Deadline constraints (`deadline_at - now` must fall in this window per the
// /jobs contract). Polling cadence β start at the response's `retry_after_ms`
// floor, exponential backoff capped at POLL_CAP_MS. Polling deadline budget
// = computed `deadline_at` + this buffer (defense in depth on top of the
// worker's own expiration handling).
export const MIN_DEADLINE_MS = 5_000;
export const MAX_DEADLINE_MS = 30 * 60 * 1_000;
export const POLL_FLOOR_MS = 1_000;
export const POLL_CAP_MS = 5_000;
// Long-poll (`wait_ms`). A wait shorter than LONG_POLL_MIN_WAIT_MS isn't
// worth holding a connection for, so the last stretch before the polling
// deadline runs on the regular schedule. A non-terminal answer that came
// back in under LONG_POLL_HELD_FRACTION of the requested wait means the
// server did not hold it (older build, or its wake-up path unavailable):
// the job drops back to the regular schedule instead of re-polling at once.
// LONG_POLL_MAX_WAIT_MS mirrors fire-pdf's default cap
// (FIRE_PDF_JOBS_LONG_POLL_MAX_WAIT_MS): fire-pdf clamps longer waits, so
// asking for more would only make a held answer look early. After
// LONG_POLL_MAX_EARLY_ANSWERS early answers in a row the job stays on the
// regular schedule; a single early answer is followed by one scheduled
// pause, never an immediate re-poll.
export const LONG_POLL_MIN_WAIT_MS = 1_000;
export const LONG_POLL_MAX_WAIT_MS = 25_000;
export const LONG_POLL_DEADLINE_SLACK_MS = 1_000;
export const LONG_POLL_HELD_FRACTION = 0.5;
export const LONG_POLL_MAX_EARLY_ANSWERS = 2;
export const POLL_TIMEOUT_BUFFER_MS = 30_000;
// An inline job's deadline sits this far inside the caller's window. The
// caller aborts the scrape the instant its window closes, so a job that is
// still wrapping up at that instant is cancelled and every processed page is
// lost. The margin covers the worker's end-of-deadline work (degrade the
// remaining pages, assemble, upload) plus one poll interval and the result
// fetch, so the deadline-degraded result the worker produces anyway is the
// one the caller receives: 10% of the window, floored at 10s (a backoff-capped
// poll alone is ~6s with jitter) and capped at 30s.
export const INLINE_JOB_DEADLINE_MARGIN_FRACTION = 0.1;
export const INLINE_JOB_DEADLINE_MARGIN_MIN_MS = 10_000;
export const INLINE_JOB_DEADLINE_MARGIN_MAX_MS = 30_000;
// Polls are pulled forward so one lands this long after the job deadline,
// then run at POLL_FLOOR_MS until the caller window closes: the job is
// expected to finish right there, and a backoff-capped poll would miss it.
export const JOB_DEADLINE_POLL_GRACE_MS = 1_000;
// One retry on a submit that provably never reached fire-pdf's handler: a
// transport failure, or a 503 carrying Fastify's canned shutdown body
// instead of one of fire-pdf's own 503 codes. Rolling api pods leave
// kept-alive connections pointed at terminating pods; the retry opens a
// fresh connection to a live one. POST /jobs is idempotent on scrape_id, so
// a first request that did land is replayed, never duplicated.
export const SUBMIT_TRANSIENT_RETRY_DELAY_MS = 250;
// Slack for the submit round trip: fire-pdf validates `deadline_at - now`
// against MIN_DEADLINE_MS on arrival, so the advertised deadline must clear
// it by at least the request's flight time.
const INLINE_SUBMIT_SLACK_MS = 5_000;
// The smallest caller window async accepts. Below it, the margin would push
// the inline job deadline to (or under) fire-pdf's minimum and the submit
// would be rejected on arrival; such requests take the sync path instead.
export const MIN_ASYNC_CALLER_WINDOW_MS =
MIN_DEADLINE_MS + INLINE_JOB_DEADLINE_MARGIN_MIN_MS + INLINE_SUBMIT_SLACK_MS;
export const TERMINAL_STATUSES = new Set([
"done",
"failed",
"expired",
"cancelled",
]);
/** The 503 codes fire-pdf's submit handlers document (api/src/http/handlers
* handle-submit-job / handle-sharded-submit). Closed on purpose: a 503 with
* any other body did not come from one of them, so it is retried once. If
* fire-pdf adds a code before this list learns it, the failure mode is one
* idempotent extra POST β never a swallowed retry. */
export const FIRE_PDF_SUBMIT_503_CODES = [
"admission_rejected",
"admission_unavailable",
"submit_preflight_failed",
"submit_txn_failed",
"page_markdown_not_ready",
"gcs_upload_failed",
"gcs_head_failed",
"lookup_failed",
"internal",
] as const;
/** Error body a fire-pdf submit handler sends with a 503: one of its
* documented codes plus a human message. */
export const firePdfSubmit503BodySchema = z
.object({
error: z.enum(FIRE_PDF_SUBMIT_503_CODES),
message: z.string().optional(),
})
.passthrough();
/** Fastify's canned reply while an instance is shutting down
* (`return503OnClosing`, sent with `Connection: close` before any route
* handler runs): the request was never processed. */
export const fastifyClosingBodySchema = z
.object({
error: z.literal("Service Unavailable"),
statusCode: z.literal(503),
})
.passthrough();
export const submitResponseSchema = z.object({
scrape_id: z.string(),
status: z.enum(["queued", "published", "running", "done"]),
lane: z
.enum(["fast", "standard", "heavy", "xl", "unknown"])
.optional()
.default("unknown"),
retry_after_ms: z.number().optional(),
});
export const pollResponseSchema = z.object({
scrape_id: z.string(),
status: z.enum([
"queued",
"published",
"running",
"done",
"failed",
"expired",
"cancelled",
]),
retry_after_ms: z.number().optional(),
pages_processed: z.number().optional(),
// Live remaining-time estimate computed server-side from lane
// throughput + backlog (fire-pdf phase 2). Optional and additive;
// absent on older fire-pdf builds or when the lane has no measured
// throughput. `.catch(undefined)` so a malformed value (negative,
// NaN, Infinity) degrades to "no estimate" instead of failing the
// whole poll response.
estimated_remaining_ms: z
.number()
.finite()
.nonnegative()
.optional()
.catch(undefined),
failed_pages: z.array(z.number()).nullable().optional(),
partial_pages: z.array(z.number()).nullable().optional(),
error_class: z.string().optional(),
error_message: z.string().optional(),
});
/**
* Physical page markdown payload, shared by the sync `/ocr` and async
* `/jobs/:id/result` response schemas. When `include_blocks` is requested
* without `include_page_markdown`, fire-pdf returns `pages` in a legacy
* block-alias shape (`{page, width, height, status, blocks}` β no markdown).
* The union matches that alias explicitly and drops it to `undefined`, so
* genuine protocol corruption still fails the response parse, and the
* callers' "requested page markdown missing" checks still fire when page
* markdown was requested but absent.
*/
export const firePdfPagesSchema = z
.union([
z.array(
z.object({ page: z.number().int().positive(), markdown: z.string() }),
),
z
.array(
// Full documented alias shape (LegacyPageBlocks) β requiring every
// field keeps the union discriminating: a payload that is neither
// valid page markdown nor a complete alias fails the parse.
z.object({
page: z.number().int().positive(),
width: z.number().nullable(),
height: z.number().nullable(),
status: z.string(),
blocks: z.array(z.unknown()),
}),
)
.transform((): undefined => undefined),
])
.optional();
/** Typed layout blocks (fire-pdf docs/blocks-schema.md), present only when
* the request set `include_blocks`. Wire shape β snake_case passthrough.
* Single source of truth for the block contract: the response parsers and
* the GCS cache validator both use it, and the wire TS types are inferred
* from it. */
export const firePdfBlockPagesSchema = z.array(
z.object({
page: z.number().int().positive(),
width: z.number().nullable(),
height: z.number().nullable(),
// Documented values: ok | partial | failed. Kept open so a new
// page status never fails an otherwise-valid scrape.
status: z.string(),
items: z.array(
z.object({
id: z.string(),
type: z.string(),
label: z.string().nullable(),
bbox: z
.tuple([z.number(), z.number(), z.number(), z.number()])
.nullable(),
content: z.string(),
markdown_span: z.tuple([z.number(), z.number()]).nullable(),
reading_order: z.number(),
source: z.string().nullable(),
confidence: z.object({
layout: z.number().nullable(),
ocr: z.number().nullable(),
}),
}),
),
}),
);
export const firePdfBlocksSchema = firePdfBlockPagesSchema.optional();
/**
* fire-pdf's provenance stamp: who produced a result and how complete it is.
* Stored verbatim with every cache entry so a later cache policy can judge
* the entry without reading its content (fire-pdf docs/cache-policy.md).
* `passthrough` keeps fields a newer fire-pdf adds.
*/
export const firePdfProvenanceSchema = z
.object({
generation: z.string(),
build_sha: z.string(),
built_at: z.string().nullable(),
produced_at: z.string(),
stages: z.array(z.string()).optional(),
// Page counts: the write rule reads them, so a malformed stamp must
// fail validation rather than pass as a healthy result. parseProvenance
// turns that failure into a refused cache write, never a failed scrape.
quality: z
.object({
total_pages: z.int().nonnegative(),
failed_pages: z.int().nonnegative(),
partial_pages: z.int().nonnegative(),
degraded_pages: z.int().nonnegative(),
ocr_pages: z.int().nonnegative(),
})
.passthrough()
.optional(),
// `passthrough` on each item too: a newer fire-pdf may add per-build
// fields, and the entry stores the stamp verbatim.
contributing_builds: z
.array(
z
.object({
generation: z.string(),
build_sha: z.string(),
built_at: z.string().nullable(),
})
.passthrough(),
)
.optional(),
})
.passthrough();
export type FirePdfProvenance = z.infer<typeof firePdfProvenanceSchema>;
/**
* The stamp is parsed apart from the document. The document is what the
* caller asked for; the stamp only decides what the cache may remember, so
* a stamp this build cannot read degrades to `malformed` (the result is
* served, not cached) instead of failing the response.
*/
type ProvenanceParse =
| { status: "absent" }
| { status: "ok"; provenance: FirePdfProvenance }
| { status: "malformed"; issue: string };
export function parseProvenance(raw: unknown): ProvenanceParse {
// Only a missing field is "no stamp" (a build from before the stamp
// existed). fire-pdf never sends an explicit null; one is unreadable.
if (raw === undefined) return { status: "absent" };
if (raw === null) return { status: "malformed", issue: "provenance: null" };
const parsed = firePdfProvenanceSchema.safeParse(raw);
if (parsed.success) return { status: "ok", provenance: parsed.data };
return {
status: "malformed",
issue: parsed.error.issues
.slice(0, 3)
.map(i => `${i.path.join(".") || "<root>"}: ${i.message}`)
.join("; "),
};
}
export const resultResponseSchema = z.object({
schema_version: z
.union([z.literal(1), z.literal(2), z.literal(3)])
.optional(),
markdown: z.string(),
pages: firePdfPagesSchema,
blocks: firePdfBlocksSchema,
pages_processed: z.number().optional(),
failed_pages: z.array(z.number()).nullable().optional(),
partial_pages: z.array(z.number()).nullable().optional(),
// Echo of an honored pageMarkers job option. Markers are baked into
// `markdown` and their absence is not detectable by content, so the echo
// is the only proof the fire-pdf worker build understood the option β
// older workers ignore unknown option keys and omit it.
page_markers: z.literal(true).optional(),
// Raw on purpose: parsed separately by parseProvenance so a stamp this
// build does not understand never fails the scrape.
provenance: z.unknown().optional(),
data_region_applied: z.unknown().optional(),
reserved_applied: z.unknown().optional(),
});
export type PollResponse = z.infer<typeof pollResponseSchema>;
export type ResultResponse = z.infer<typeof resultResponseSchema>;