Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(api): fire-pdf async fails loudly instead of silent sync fallback (#3543)
Abimael Martell05994928d424
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.tsindex dca6581f7..a76ba809d 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts@@ -7,7 +7,10 @@ jest.mock("../../../../../lib/gcs-pdf-cache", () => ({savePdfResultToCache: jest.fn(async () => null),}));-import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";+import {+ FirePdfAsyncFailure,+ scrapePDFWithFirePDFAsync,+} from "../fire-pdf/async";import { config } from "../../../../../config";// ββ Fixtures βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ@@ -219,59 +222,56 @@ describe("scrapePDFWithFirePDFAsync", () => {});it.each([- ["404", 404],- ["413", 413],- ["429", 429],- ["503", 503],- ["generic 5xx", 500],- ])("falls back to sync when POST /jobs returns %s", async (_, status) => {- const { fetchImpl } = makeFetchFromSequence([- {- matchUrl: /\/jobs$/,- matchMethod: "POST",- response: { status, body: { error: "x" } },- },- ]);- const fallback = jest.fn(async () => ({- markdown: "sync-result",- html: "<p>sync-result</p>",- pagesProcessed: 1,- }));+ ["404", 404, "http_404"],+ ["413", 413, "http_413"],+ ["429", 429, "http_429"],+ ["503", 503, "http_503"],+ ["generic 5xx", 500, "http_5xx"],+ ])(+ "throws FirePdfAsyncFailure when POST /jobs returns %s",+ async (_, status, reason) => {+ const { fetchImpl } = makeFetchFromSequence([+ {+ matchUrl: /\/jobs$/,+ matchMethod: "POST",+ response: { status, body: { error: "x" } },+ },+ ]);+ const fallback = jest.fn();- const result = await scrapePDFWithFirePDFAsync(- makeMeta(),- "BASE64",- undefined,- undefined,- undefined,- { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },- );+ const err = await scrapePDFWithFirePDFAsync(+ makeMeta(),+ "BASE64",+ undefined,+ undefined,+ undefined,+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },+ ).catch(e => e);- expect(fallback).toHaveBeenCalledTimes(1);- expect(result.markdown).toBe("sync-result");- });+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);+ expect(err.reason).toBe(reason);+ expect(fallback).not.toHaveBeenCalled();+ },+ );- it("falls back to sync when POST /jobs throws a network error", async () => {+ it("throws FirePdfAsyncFailure when POST /jobs throws a network error", async () => {const fetchImpl: any = async () => {throw new Error("connect ECONNREFUSED");};- const fallback = jest.fn(async () => ({- markdown: "sync",- html: "",- pagesProcessed: 0,- }));+ const fallback = jest.fn();- const result = await scrapePDFWithFirePDFAsync(+ const err = await scrapePDFWithFirePDFAsync(makeMeta(),"BASE64",undefined,undefined,undefined,{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },- );+ ).catch(e => e);- expect(fallback).toHaveBeenCalledTimes(1);- expect(result.markdown).toBe("sync");+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);+ expect(err.reason).toBe("network_error");+ expect(fallback).not.toHaveBeenCalled();});it("throws on POST 409 scrape_id conflict (fatal, no fallback)", async () => {@@ -300,7 +300,7 @@ describe("scrapePDFWithFirePDFAsync", () => {expect(fallback).not.toHaveBeenCalled();});- it("falls back when polling returns terminal failed (502)", async () => {+ it("throws FirePdfAsyncFailure when polling returns terminal failed (502)", async () => {const { fetchImpl } = makeFetchFromSequence([{matchUrl: /\/jobs$/,@@ -322,26 +322,23 @@ describe("scrapePDFWithFirePDFAsync", () => {},},]);- const fallback = jest.fn(async () => ({- markdown: "sync",- html: "",- pagesProcessed: 0,- }));+ const fallback = jest.fn();- const result = await scrapePDFWithFirePDFAsync(+ const err = await scrapePDFWithFirePDFAsync(makeMeta(),"BASE64",undefined,undefined,undefined,{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },- );+ ).catch(e => e);- expect(fallback).toHaveBeenCalledTimes(1);- expect(result.markdown).toBe("sync");+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);+ expect(err.reason).toBe("terminal_failed");+ expect(fallback).not.toHaveBeenCalled();});- it("falls back when polling returns 410 (expired/cancelled)", async () => {+ it("throws FirePdfAsyncFailure when polling returns 410 (expired)", async () => {const { fetchImpl } = makeFetchFromSequence([{matchUrl: /\/jobs$/,@@ -358,26 +355,23 @@ describe("scrapePDFWithFirePDFAsync", () => {},},]);- const fallback = jest.fn(async () => ({- markdown: "sync",- html: "",- pagesProcessed: 0,- }));+ const fallback = jest.fn();- const result = await scrapePDFWithFirePDFAsync(+ const err = await scrapePDFWithFirePDFAsync(makeMeta(),"BASE64",undefined,undefined,undefined,{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },- );+ ).catch(e => e);- expect(fallback).toHaveBeenCalledTimes(1);- expect(result.markdown).toBe("sync");+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);+ expect(err.reason).toBe("terminal_expired");+ expect(fallback).not.toHaveBeenCalled();});- it("falls back when polling exceeds deadline + buffer", async () => {+ it("throws FirePdfAsyncFailure when polling exceeds deadline + buffer", async () => {let virtualNow = 1_000_000;const advance = (ms: number) => {virtualNow += ms;@@ -402,11 +396,7 @@ describe("scrapePDFWithFirePDFAsync", () => {response: { status: 202, body: { scrape_id: "x", status: "running" } },},]);- const fallback = jest.fn(async () => ({- markdown: "sync",- html: "",- pagesProcessed: 0,- }));+ const fallback = jest.fn();// 5s scrape budget β deadline 5s, polling deadline = submit + 5s + 30s = 35s.// Each sleep advances time by 60s, blowing past the polling deadline.@@ -418,7 +408,7 @@ describe("scrapePDFWithFirePDFAsync", () => {},});- const result = await scrapePDFWithFirePDFAsync(+ const err = await scrapePDFWithFirePDFAsync(meta,"BASE64",undefined,@@ -430,13 +420,14 @@ describe("scrapePDFWithFirePDFAsync", () => {sleepImpl: async ms => advance(ms + 60_000),nowImpl: () => virtualNow,},- );+ ).catch(e => e);- expect(fallback).toHaveBeenCalledTimes(1);- expect(result.markdown).toBe("sync");+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);+ expect(err.reason).toBe("polling_timeout");+ expect(fallback).not.toHaveBeenCalled();});- it("falls back when result endpoint returns 503", async () => {+ it("throws FirePdfAsyncFailure when result endpoint returns 503", async () => {const { fetchImpl } = makeFetchFromSequence([{matchUrl: /\/jobs$/,@@ -457,23 +448,20 @@ describe("scrapePDFWithFirePDFAsync", () => {response: { status: 503, body: { error: "gcs_unreachable" } },},]);- const fallback = jest.fn(async () => ({- markdown: "sync",- html: "",- pagesProcessed: 0,- }));+ const fallback = jest.fn();- const result = await scrapePDFWithFirePDFAsync(+ const err = await scrapePDFWithFirePDFAsync(makeMeta(),"BASE64",undefined,undefined,undefined,{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },- );+ ).catch(e => e);- expect(fallback).toHaveBeenCalledTimes(1);- expect(result.markdown).toBe("sync");+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);+ expect(err.reason).toBe("result_503");+ expect(fallback).not.toHaveBeenCalled();});it("re-polls once on result 409, then succeeds", async () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.tsindex c9d8a25eb..7c4f6c0b9 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts@@ -7,16 +7,14 @@ import { safeMarkdownToHtml } from "../markdownToHtml";import { scrapePDFWithFirePDF } from "../firePDF";import { firePdfAsyncTotalDurationSeconds } from "./metrics";import { POLL_FLOOR_MS, POLL_TIMEOUT_BUFFER_MS } from "./schema";-import {- defaultSleep,- computeDeadlineMs,- makeFallback,-} from "./utils";+import { defaultSleep, computeDeadlineMs } from "./utils";import { tryGetCached, maybeSaveResult } from "./cache";import { submitJob } from "./submit";-import { pollUntilTerminal, type PollOutcome } from "./poll";+import { pollUntilTerminal } from "./poll";import { fetchResult } from "./result";+export { FirePdfAsyncFailure } from "./utils";+type FirePdfAsyncDeps = {fetchImpl?: typeof undiciFetch;fallbackImpl?: typeof scrapePDFWithFirePDF;@@ -61,15 +59,6 @@ export async function scrapePDFWithFirePDFAsync(const deadlineAt = new Date(submitTime + deadlineFromNow).toISOString();const pollingDeadline = submitTime + deadlineFromNow + POLL_TIMEOUT_BUFFER_MS;- const fallback = makeFallback({- meta,- fallbackImpl,- base64Content,- maxPages,- pagesProcessed,- mode,- });-// ββ Step 1: POST /jobs ββββββββββββββββββββββββββββββββββββββββββββββββconst submit = await submitJob({meta,@@ -80,15 +69,12 @@ export async function scrapePDFWithFirePDFAsync(mode,deadlineAt,fetchImpl,- fallback,});- if (submit.kind === "fallback") return submit.result;// ββ Step 2: poll until terminal (skip on idempotent-replay done) ββββββ- const polled: PollOutcome = submit.alreadyDone+ const polled = submit.alreadyDone? {- kind: "done",- poll: { scrape_id: meta.id, status: "done" },+ poll: { scrape_id: meta.id, status: "done" as const },pollCount: 0,}: await pollUntilTerminal({@@ -100,9 +86,7 @@ export async function scrapePDFWithFirePDFAsync(fetchImpl,sleep,now,- fallback,});- if (polled.kind === "fallback") return polled.result;// ββ Step 3: GET /jobs/:id/result ββββββββββββββββββββββββββββββββββββββconst fetched = await fetchResult({@@ -111,35 +95,27 @@ export async function scrapePDFWithFirePDFAsync(meta,fetchImpl,sleep,- fallback,});- if (fetched.kind === "fallback") return fetched.result;// ββ Assemble + cache save βββββββββββββββββββββββββββββββββββββββββββββconst pages =- fetched.result.pages_processed ??- polled.poll.pages_processed ??- pagesProcessed;+ fetched.pages_processed ?? polled.poll.pages_processed ?? pagesProcessed;const durationMs = now() - overallStartedAt;firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);meta.logger.info("FirePDF async completed", {scrapeId: meta.id,durationMs,- markdownLength: fetched.result.markdown.length,+ markdownLength: fetched.markdown.length,pagesProcessed: pages,- failedPages: fetched.result.failed_pages,- partialPages: fetched.result.partial_pages,+ failedPages: fetched.failed_pages,+ partialPages: fetched.partial_pages,pollCount: polled.pollCount,});const processorResult: PDFProcessorResult & { markdown: string } = {- markdown: fetched.result.markdown,- html: await safeMarkdownToHtml(- fetched.result.markdown,- meta.logger,- meta.id,- ),+ markdown: fetched.markdown,+ html: await safeMarkdownToHtml(fetched.markdown, meta.logger, meta.id),pagesProcessed: pages,};diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.tsindex 90c846baf..e2592792c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts@@ -1,5 +1,4 @@import type { Meta } from "../../..";-import type { PDFProcessorResult } from "../types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import {@@ -12,9 +11,9 @@ import {TERMINAL_STATUSES,type PollResponse,} from "./schema";-import { nextPollDelay, type Fallback } from "./utils";+import { failAsync, nextPollDelay } from "./utils";-export type PollDeps = {+type PollDeps = {baseUrl: string;scrapeId: string;initialDelay: number;@@ -23,16 +22,11 @@ export type PollDeps = {fetchImpl: typeof undiciFetch;sleep: (ms: number, signal?: AbortSignal) => Promise<void>;now: () => number;- fallback: Fallback;};-export type PollOutcome =- | { kind: "done"; poll: PollResponse; pollCount: number }- | { kind: "fallback"; result: PDFProcessorResult };+type PollOk = { poll: PollResponse; pollCount: number };-export async function pollUntilTerminal(- deps: PollDeps,-): Promise<PollOutcome> {+export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {const {baseUrl,scrapeId,@@ -41,7 +35,6 @@ export async function pollUntilTerminal(fetchImpl,sleep,now,- fallback,} = deps;let pollCount = 0;let lastDelay = deps.initialDelay;@@ -49,10 +42,7 @@ export async function pollUntilTerminal(while (true) {if (now() > pollingDeadline) {firePdfAsyncPollCount.observe(pollCount);- return {- kind: "fallback",- result: await fallback("polling_timeout", { pollCount }),- };+ failAsync(meta, "polling_timeout", { pollCount });}meta.abort.throwIfAborted();@@ -68,13 +58,10 @@ export async function pollUntilTerminal(} catch (error) {if (error instanceof AbortManagerThrownError) throw error;firePdfAsyncPollCount.observe(pollCount);- return {- kind: "fallback",- result: await fallback("network_error", {- error: String(error),- pollCount,- }),- };+ failAsync(meta, "network_error", {+ error: String(error),+ pollCount,+ });}const pollStatus = pollResp.status;@@ -92,50 +79,39 @@ export async function pollUntilTerminal(const parsed = pollResponseSchema.safeParse(pollBody);const status = parsed.success ? parsed.data.status : "expired";firePdfAsyncCompletedTotal.labels(status).inc();- return {- kind: "fallback",- result: await fallback(- status === "cancelled" ? "terminal_cancelled" : "terminal_expired",- { status, pollCount, body: pollBody },- ),- };+ failAsync(+ meta,+ status === "cancelled" ? "terminal_cancelled" : "terminal_expired",+ { status, pollCount, body: pollBody },+ );}if (pollStatus === 502) {firePdfAsyncPollCount.observe(pollCount);firePdfAsyncCompletedTotal.labels("failed").inc();- return {- kind: "fallback",- result: await fallback("terminal_failed", {- pollCount,- body: pollBody,- }),- };+ failAsync(meta, "terminal_failed", {+ pollCount,+ body: pollBody,+ });}if (pollStatus !== 200 && pollStatus !== 202) {firePdfAsyncPollCount.observe(pollCount);- return {- kind: "fallback",- result: await fallback("http_5xx", {- status: pollStatus,- body: pollBody,- pollCount,- }),- };+ failAsync(meta, "http_5xx", {+ status: pollStatus,+ body: pollBody,+ pollCount,+ });}const parsed = pollResponseSchema.safeParse(pollBody);if (!parsed.success) {firePdfAsyncPollCount.observe(pollCount);- return {- kind: "fallback",- result: await fallback("http_5xx", {- error: String(parsed.error),- body: pollBody,- pollCount,- }),- };+ failAsync(meta, "http_5xx", {+ error: String(parsed.error),+ body: pollBody,+ pollCount,+ });}if (TERMINAL_STATUSES.has(parsed.data.status)) {@@ -148,17 +124,14 @@ export async function pollUntilTerminal(: parsed.data.status === "expired"? "terminal_expired": "terminal_cancelled";- return {- kind: "fallback",- result: await fallback(reason, {- status: parsed.data.status,- errorClass: parsed.data.error_class,- errorMessage: parsed.data.error_message,- pollCount,- }),- };+ failAsync(meta, reason, {+ status: parsed.data.status,+ errorClass: parsed.data.error_class,+ errorMessage: parsed.data.error_message,+ pollCount,+ });}- return { kind: "done", poll: parsed.data, pollCount };+ return { poll: parsed.data, pollCount };}lastDelay = nextPollDelay(lastDelay, parsed.data.retry_after_ms);diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.tsindex 633b19ecd..09a11a8c3 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts@@ -1,5 +1,4 @@import type { Meta } from "../../..";-import type { PDFProcessorResult } from "../types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import {@@ -7,23 +6,18 @@ import {resultResponseSchema,type ResultResponse,} from "./schema";-import type { Fallback } from "./utils";+import { failAsync } from "./utils";-export type ResultDeps = {+type ResultDeps = {baseUrl: string;scrapeId: string;meta: Meta;fetchImpl: typeof undiciFetch;sleep: (ms: number, signal?: AbortSignal) => Promise<void>;- fallback: Fallback;};-export type ResultOutcome =- | { kind: "ok"; result: ResultResponse }- | { kind: "fallback"; result: PDFProcessorResult };--export async function fetchResult(deps: ResultDeps): Promise<ResultOutcome> {- const { baseUrl, scrapeId, meta, fetchImpl, sleep, fallback } = deps;+export async function fetchResult(deps: ResultDeps): Promise<ResultResponse> {+ const { baseUrl, scrapeId, meta, fetchImpl, sleep } = deps;let retried409 = 0;while (true) {@@ -35,33 +29,24 @@ export async function fetchResult(deps: ResultDeps): Promise<ResultOutcome> {});} catch (error) {if (error instanceof AbortManagerThrownError) throw error;- return {- kind: "fallback",- result: await fallback("network_error", { error: String(error) }),- };+ failAsync(meta, "network_error", { error: String(error) });}const status = resp.status;const body = await resp.json().catch(() => ({}));if (status === 503) {- return {- kind: "fallback",- result: await fallback("result_503", { body }),- };+ failAsync(meta, "result_503", { body });}if (status === 409) {retried409++;if (retried409 > 1) {- return {- kind: "fallback",- result: await fallback("http_5xx", {- status: 409,- body,- note: "result endpoint kept returning 409",- }),- };+ failAsync(meta, "http_5xx", {+ status: 409,+ body,+ note: "result endpoint kept returning 409",+ });}meta.logger.info("FirePDF async result returned 409, re-polling once", {scrapeId,@@ -71,22 +56,16 @@ export async function fetchResult(deps: ResultDeps): Promise<ResultOutcome> {}if (status !== 200) {- return {- kind: "fallback",- result: await fallback("http_5xx", { status, body }),- };+ failAsync(meta, "http_5xx", { status, body });}const parsed = resultResponseSchema.safeParse(body);if (!parsed.success) {- return {- kind: "fallback",- result: await fallback("http_5xx", {- error: String(parsed.error),- body,- }),- };+ failAsync(meta, "http_5xx", {+ error: String(parsed.error),+ body,+ });}- return { kind: "ok", result: parsed.data };+ return parsed.data;}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex e239ed31c..220b6d769 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -1,22 +1,18 @@import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";-import type { PDFProcessorResult } from "../types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";import { firePdfAsyncSubmittedTotal } from "./metrics";import { submitResponseSchema } from "./schema";-import type { Fallback } from "./utils";+import { failAsync } from "./utils";-export type SubmitOutcome =- | {- kind: "ok";- lane: string | undefined;- retryAfterMs: number | undefined;- alreadyDone: boolean;- }- | { kind: "fallback"; result: PDFProcessorResult };+type SubmitOutcome = {+ lane: string | undefined;+ retryAfterMs: number | undefined;+ alreadyDone: boolean;+};-export type SubmitArgs = {+type SubmitArgs = {meta: Meta;baseUrl: string;base64Content: string;@@ -25,7 +21,6 @@ export type SubmitArgs = {mode: PDFMode | undefined;deadlineAt: string;fetchImpl: typeof undiciFetch;- fallback: Fallback;};export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {@@ -38,7 +33,6 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {mode,deadlineAt,fetchImpl,- fallback,} = args;const scrapeId = meta.id;@@ -74,13 +68,13 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {json = await resp.json().catch(() => ({}));} catch (error) {if (error instanceof AbortManagerThrownError) throw error;- return { kind: "fallback", result: await fallback("network_error", { error: String(error) }) };+ failAsync(meta, "network_error", { error: String(error) });}- if (status === 404) return { kind: "fallback", result: await fallback("http_404") };- if (status === 413) return { kind: "fallback", result: await fallback("http_413") };- if (status === 429) return { kind: "fallback", result: await fallback("http_429") };- if (status === 503) return { kind: "fallback", result: await fallback("http_503") };+ if (status === 404) failAsync(meta, "http_404");+ if (status === 413) failAsync(meta, "http_413");+ if (status === 429) failAsync(meta, "http_429");+ if (status === 503) failAsync(meta, "http_503");if (status === 409) {meta.logger.error("FirePDF async POST /jobs returned 409 scrape_id_conflict", {@@ -101,22 +95,16 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {}if (status !== 200 && status !== 202) {- return {- kind: "fallback",- result: await fallback("http_5xx", { status, body: json }),- };+ failAsync(meta, "http_5xx", { status, body: json });}const parsed = submitResponseSchema.safeParse(json);if (!parsed.success) {- return {- kind: "fallback",- result: await fallback("http_5xx", {- error: String(parsed.error),- body: json,- status,- }),- };+ failAsync(meta, "http_5xx", {+ error: String(parsed.error),+ body: json,+ status,+ });}firePdfAsyncSubmittedTotal.labels(parsed.data.lane ?? "unknown").inc();@@ -129,7 +117,6 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {});return {- kind: "ok",lane: parsed.data.lane,retryAfterMs: parsed.data.retry_after_ms,alreadyDone: status === 200 && parsed.data.status === "done",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex a96b75cab..f263c35d8 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -1,7 +1,4 @@import type { Meta } from "../../..";-import type { PDFMode } from "../../../../../controllers/v2/types";-import type { PDFProcessorResult } from "../types";-import type { scrapePDFWithFirePDF } from "../firePDF";import {MIN_DEADLINE_MS,MAX_DEADLINE_MS,@@ -59,31 +56,26 @@ export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {return Math.min(MAX_DEADLINE_MS, Math.max(MIN_DEADLINE_MS, candidate));}-export type Fallback = (- reason: FallbackReason,- extra?: Record<string, unknown>,-) => Promise<PDFProcessorResult>;--export type FallbackCtx = {- meta: Meta;- fallbackImpl: typeof scrapePDFWithFirePDF;- base64Content: string;- maxPages?: number;- pagesProcessed?: number;- mode?: PDFMode;-};+export class FirePdfAsyncFailure extends Error {+ constructor(+ public readonly reason: FallbackReason,+ public readonly extra: Record<string, unknown> = {},+ ) {+ super(`fire-pdf async failed: ${reason}`);+ this.name = "FirePdfAsyncFailure";+ }+}-export function makeFallback(ctx: FallbackCtx): Fallback {- const { meta, fallbackImpl, base64Content, maxPages, pagesProcessed, mode } =- ctx;- const scrapeId = meta.id;- return async (reason, extra = {}) => {- firePdfAsyncFallbackTotal.labels(reason).inc();- meta.logger.warn("FirePDF async falling back to sync /ocr", {- scrapeId,- reason,- ...extra,- });- return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);- };+export function failAsync(+ meta: Meta,+ reason: FallbackReason,+ extra: Record<string, unknown> = {},+): never {+ firePdfAsyncFallbackTotal.labels(reason).inc();+ meta.logger.warn("FirePDF async failed", {+ scrapeId: meta.id,+ reason,+ ...extra,+ });+ throw new FirePdfAsyncFailure(reason, extra);}