Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

refactor(api): fire-pdf async fails loudly instead of silent sync fallback (#3543)

Abimael Martell05994928d424
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
index dca6581f7..a76ba809d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFAsync.test.ts
@@ -7,7 +7,10 @@ jest.mock("../../../../../lib/gcs-pdf-cache", () => ({
savePdfResultToCache: jest.fn(async () => null),
}));
-import { scrapePDFWithFirePDFAsync } from "../fire-pdf/async";
+import {
+ FirePdfAsyncFailure,
+ scrapePDFWithFirePDFAsync,
+} from "../fire-pdf/async";
import { config } from "../../../../../config";
// ── Fixtures ─────────────────────────────────────────────────────────────
@@ -219,59 +222,56 @@ describe("scrapePDFWithFirePDFAsync", () => {
});
it.each([
- ["404", 404],
- ["413", 413],
- ["429", 429],
- ["503", 503],
- ["generic 5xx", 500],
- ])("falls back to sync when POST /jobs returns %s", async (_, status) => {
- const { fetchImpl } = makeFetchFromSequence([
- {
- matchUrl: /\/jobs$/,
- matchMethod: "POST",
- response: { status, body: { error: "x" } },
- },
- ]);
- const fallback = jest.fn(async () => ({
- markdown: "sync-result",
- html: "<p>sync-result</p>",
- pagesProcessed: 1,
- }));
+ ["404", 404, "http_404"],
+ ["413", 413, "http_413"],
+ ["429", 429, "http_429"],
+ ["503", 503, "http_503"],
+ ["generic 5xx", 500, "http_5xx"],
+ ])(
+ "throws FirePdfAsyncFailure when POST /jobs returns %s",
+ async (_, status, reason) => {
+ const { fetchImpl } = makeFetchFromSequence([
+ {
+ matchUrl: /\/jobs$/,
+ matchMethod: "POST",
+ response: { status, body: { error: "x" } },
+ },
+ ]);
+ const fallback = jest.fn();
- const result = await scrapePDFWithFirePDFAsync(
- makeMeta(),
- "BASE64",
- undefined,
- undefined,
- undefined,
- { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
- );
+ const err = await scrapePDFWithFirePDFAsync(
+ makeMeta(),
+ "BASE64",
+ undefined,
+ undefined,
+ undefined,
+ { fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
+ ).catch(e => e);
- expect(fallback).toHaveBeenCalledTimes(1);
- expect(result.markdown).toBe("sync-result");
- });
+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(err.reason).toBe(reason);
+ expect(fallback).not.toHaveBeenCalled();
+ },
+ );
- it("falls back to sync when POST /jobs throws a network error", async () => {
+ it("throws FirePdfAsyncFailure when POST /jobs throws a network error", async () => {
const fetchImpl: any = async () => {
throw new Error("connect ECONNREFUSED");
};
- const fallback = jest.fn(async () => ({
- markdown: "sync",
- html: "",
- pagesProcessed: 0,
- }));
+ const fallback = jest.fn();
- const result = await scrapePDFWithFirePDFAsync(
+ const err = await scrapePDFWithFirePDFAsync(
makeMeta(),
"BASE64",
undefined,
undefined,
undefined,
{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
- );
+ ).catch(e => e);
- expect(fallback).toHaveBeenCalledTimes(1);
- expect(result.markdown).toBe("sync");
+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(err.reason).toBe("network_error");
+ expect(fallback).not.toHaveBeenCalled();
});
it("throws on POST 409 scrape_id conflict (fatal, no fallback)", async () => {
@@ -300,7 +300,7 @@ describe("scrapePDFWithFirePDFAsync", () => {
expect(fallback).not.toHaveBeenCalled();
});
- it("falls back when polling returns terminal failed (502)", async () => {
+ it("throws FirePdfAsyncFailure when polling returns terminal failed (502)", async () => {
const { fetchImpl } = makeFetchFromSequence([
{
matchUrl: /\/jobs$/,
@@ -322,26 +322,23 @@ describe("scrapePDFWithFirePDFAsync", () => {
},
},
]);
- const fallback = jest.fn(async () => ({
- markdown: "sync",
- html: "",
- pagesProcessed: 0,
- }));
+ const fallback = jest.fn();
- const result = await scrapePDFWithFirePDFAsync(
+ const err = await scrapePDFWithFirePDFAsync(
makeMeta(),
"BASE64",
undefined,
undefined,
undefined,
{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
- );
+ ).catch(e => e);
- expect(fallback).toHaveBeenCalledTimes(1);
- expect(result.markdown).toBe("sync");
+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(err.reason).toBe("terminal_failed");
+ expect(fallback).not.toHaveBeenCalled();
});
- it("falls back when polling returns 410 (expired/cancelled)", async () => {
+ it("throws FirePdfAsyncFailure when polling returns 410 (expired)", async () => {
const { fetchImpl } = makeFetchFromSequence([
{
matchUrl: /\/jobs$/,
@@ -358,26 +355,23 @@ describe("scrapePDFWithFirePDFAsync", () => {
},
},
]);
- const fallback = jest.fn(async () => ({
- markdown: "sync",
- html: "",
- pagesProcessed: 0,
- }));
+ const fallback = jest.fn();
- const result = await scrapePDFWithFirePDFAsync(
+ const err = await scrapePDFWithFirePDFAsync(
makeMeta(),
"BASE64",
undefined,
undefined,
undefined,
{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
- );
+ ).catch(e => e);
- expect(fallback).toHaveBeenCalledTimes(1);
- expect(result.markdown).toBe("sync");
+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(err.reason).toBe("terminal_expired");
+ expect(fallback).not.toHaveBeenCalled();
});
- it("falls back when polling exceeds deadline + buffer", async () => {
+ it("throws FirePdfAsyncFailure when polling exceeds deadline + buffer", async () => {
let virtualNow = 1_000_000;
const advance = (ms: number) => {
virtualNow += ms;
@@ -402,11 +396,7 @@ describe("scrapePDFWithFirePDFAsync", () => {
response: { status: 202, body: { scrape_id: "x", status: "running" } },
},
]);
- const fallback = jest.fn(async () => ({
- markdown: "sync",
- html: "",
- pagesProcessed: 0,
- }));
+ const fallback = jest.fn();
// 5s scrape budget β†’ deadline 5s, polling deadline = submit + 5s + 30s = 35s.
// Each sleep advances time by 60s, blowing past the polling deadline.
@@ -418,7 +408,7 @@ describe("scrapePDFWithFirePDFAsync", () => {
},
});
- const result = await scrapePDFWithFirePDFAsync(
+ const err = await scrapePDFWithFirePDFAsync(
meta,
"BASE64",
undefined,
@@ -430,13 +420,14 @@ describe("scrapePDFWithFirePDFAsync", () => {
sleepImpl: async ms => advance(ms + 60_000),
nowImpl: () => virtualNow,
},
- );
+ ).catch(e => e);
- expect(fallback).toHaveBeenCalledTimes(1);
- expect(result.markdown).toBe("sync");
+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(err.reason).toBe("polling_timeout");
+ expect(fallback).not.toHaveBeenCalled();
});
- it("falls back when result endpoint returns 503", async () => {
+ it("throws FirePdfAsyncFailure when result endpoint returns 503", async () => {
const { fetchImpl } = makeFetchFromSequence([
{
matchUrl: /\/jobs$/,
@@ -457,23 +448,20 @@ describe("scrapePDFWithFirePDFAsync", () => {
response: { status: 503, body: { error: "gcs_unreachable" } },
},
]);
- const fallback = jest.fn(async () => ({
- markdown: "sync",
- html: "",
- pagesProcessed: 0,
- }));
+ const fallback = jest.fn();
- const result = await scrapePDFWithFirePDFAsync(
+ const err = await scrapePDFWithFirePDFAsync(
makeMeta(),
"BASE64",
undefined,
undefined,
undefined,
{ fetchImpl, fallbackImpl: fallback, sleepImpl: noopSleep },
- );
+ ).catch(e => e);
- expect(fallback).toHaveBeenCalledTimes(1);
- expect(result.markdown).toBe("sync");
+ expect(err).toBeInstanceOf(FirePdfAsyncFailure);
+ expect(err.reason).toBe("result_503");
+ expect(fallback).not.toHaveBeenCalled();
});
it("re-polls once on result 409, then succeeds", async () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
index c9d8a25eb..7c4f6c0b9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/async.ts
@@ -7,16 +7,14 @@ import { safeMarkdownToHtml } from "../markdownToHtml";
import { scrapePDFWithFirePDF } from "../firePDF";
import { firePdfAsyncTotalDurationSeconds } from "./metrics";
import { POLL_FLOOR_MS, POLL_TIMEOUT_BUFFER_MS } from "./schema";
-import {
- defaultSleep,
- computeDeadlineMs,
- makeFallback,
-} from "./utils";
+import { defaultSleep, computeDeadlineMs } from "./utils";
import { tryGetCached, maybeSaveResult } from "./cache";
import { submitJob } from "./submit";
-import { pollUntilTerminal, type PollOutcome } from "./poll";
+import { pollUntilTerminal } from "./poll";
import { fetchResult } from "./result";
+export { FirePdfAsyncFailure } from "./utils";
+
type FirePdfAsyncDeps = {
fetchImpl?: typeof undiciFetch;
fallbackImpl?: typeof scrapePDFWithFirePDF;
@@ -61,15 +59,6 @@ export async function scrapePDFWithFirePDFAsync(
const deadlineAt = new Date(submitTime + deadlineFromNow).toISOString();
const pollingDeadline = submitTime + deadlineFromNow + POLL_TIMEOUT_BUFFER_MS;
- const fallback = makeFallback({
- meta,
- fallbackImpl,
- base64Content,
- maxPages,
- pagesProcessed,
- mode,
- });
-
// ── Step 1: POST /jobs ────────────────────────────────────────────────
const submit = await submitJob({
meta,
@@ -80,15 +69,12 @@ export async function scrapePDFWithFirePDFAsync(
mode,
deadlineAt,
fetchImpl,
- fallback,
});
- if (submit.kind === "fallback") return submit.result;
// ── Step 2: poll until terminal (skip on idempotent-replay done) ──────
- const polled: PollOutcome = submit.alreadyDone
+ const polled = submit.alreadyDone
? {
- kind: "done",
- poll: { scrape_id: meta.id, status: "done" },
+ poll: { scrape_id: meta.id, status: "done" as const },
pollCount: 0,
}
: await pollUntilTerminal({
@@ -100,9 +86,7 @@ export async function scrapePDFWithFirePDFAsync(
fetchImpl,
sleep,
now,
- fallback,
});
- if (polled.kind === "fallback") return polled.result;
// ── Step 3: GET /jobs/:id/result ──────────────────────────────────────
const fetched = await fetchResult({
@@ -111,35 +95,27 @@ export async function scrapePDFWithFirePDFAsync(
meta,
fetchImpl,
sleep,
- fallback,
});
- if (fetched.kind === "fallback") return fetched.result;
// ── Assemble + cache save ─────────────────────────────────────────────
const pages =
- fetched.result.pages_processed ??
- polled.poll.pages_processed ??
- pagesProcessed;
+ fetched.pages_processed ?? polled.poll.pages_processed ?? pagesProcessed;
const durationMs = now() - overallStartedAt;
firePdfAsyncTotalDurationSeconds.observe(durationMs / 1000);
meta.logger.info("FirePDF async completed", {
scrapeId: meta.id,
durationMs,
- markdownLength: fetched.result.markdown.length,
+ markdownLength: fetched.markdown.length,
pagesProcessed: pages,
- failedPages: fetched.result.failed_pages,
- partialPages: fetched.result.partial_pages,
+ failedPages: fetched.failed_pages,
+ partialPages: fetched.partial_pages,
pollCount: polled.pollCount,
});
const processorResult: PDFProcessorResult & { markdown: string } = {
- markdown: fetched.result.markdown,
- html: await safeMarkdownToHtml(
- fetched.result.markdown,
- meta.logger,
- meta.id,
- ),
+ markdown: fetched.markdown,
+ html: await safeMarkdownToHtml(fetched.markdown, meta.logger, meta.id),
pagesProcessed: pages,
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
index 90c846baf..e2592792c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/poll.ts
@@ -1,5 +1,4 @@
import type { Meta } from "../../..";
-import type { PDFProcessorResult } from "../types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import {
@@ -12,9 +11,9 @@ import {
TERMINAL_STATUSES,
type PollResponse,
} from "./schema";
-import { nextPollDelay, type Fallback } from "./utils";
+import { failAsync, nextPollDelay } from "./utils";
-export type PollDeps = {
+type PollDeps = {
baseUrl: string;
scrapeId: string;
initialDelay: number;
@@ -23,16 +22,11 @@ export type PollDeps = {
fetchImpl: typeof undiciFetch;
sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
now: () => number;
- fallback: Fallback;
};
-export type PollOutcome =
- | { kind: "done"; poll: PollResponse; pollCount: number }
- | { kind: "fallback"; result: PDFProcessorResult };
+type PollOk = { poll: PollResponse; pollCount: number };
-export async function pollUntilTerminal(
- deps: PollDeps,
-): Promise<PollOutcome> {
+export async function pollUntilTerminal(deps: PollDeps): Promise<PollOk> {
const {
baseUrl,
scrapeId,
@@ -41,7 +35,6 @@ export async function pollUntilTerminal(
fetchImpl,
sleep,
now,
- fallback,
} = deps;
let pollCount = 0;
let lastDelay = deps.initialDelay;
@@ -49,10 +42,7 @@ export async function pollUntilTerminal(
while (true) {
if (now() > pollingDeadline) {
firePdfAsyncPollCount.observe(pollCount);
- return {
- kind: "fallback",
- result: await fallback("polling_timeout", { pollCount }),
- };
+ failAsync(meta, "polling_timeout", { pollCount });
}
meta.abort.throwIfAborted();
@@ -68,13 +58,10 @@ export async function pollUntilTerminal(
} catch (error) {
if (error instanceof AbortManagerThrownError) throw error;
firePdfAsyncPollCount.observe(pollCount);
- return {
- kind: "fallback",
- result: await fallback("network_error", {
- error: String(error),
- pollCount,
- }),
- };
+ failAsync(meta, "network_error", {
+ error: String(error),
+ pollCount,
+ });
}
const pollStatus = pollResp.status;
@@ -92,50 +79,39 @@ export async function pollUntilTerminal(
const parsed = pollResponseSchema.safeParse(pollBody);
const status = parsed.success ? parsed.data.status : "expired";
firePdfAsyncCompletedTotal.labels(status).inc();
- return {
- kind: "fallback",
- result: await fallback(
- status === "cancelled" ? "terminal_cancelled" : "terminal_expired",
- { status, pollCount, body: pollBody },
- ),
- };
+ failAsync(
+ meta,
+ status === "cancelled" ? "terminal_cancelled" : "terminal_expired",
+ { status, pollCount, body: pollBody },
+ );
}
if (pollStatus === 502) {
firePdfAsyncPollCount.observe(pollCount);
firePdfAsyncCompletedTotal.labels("failed").inc();
- return {
- kind: "fallback",
- result: await fallback("terminal_failed", {
- pollCount,
- body: pollBody,
- }),
- };
+ failAsync(meta, "terminal_failed", {
+ pollCount,
+ body: pollBody,
+ });
}
if (pollStatus !== 200 && pollStatus !== 202) {
firePdfAsyncPollCount.observe(pollCount);
- return {
- kind: "fallback",
- result: await fallback("http_5xx", {
- status: pollStatus,
- body: pollBody,
- pollCount,
- }),
- };
+ failAsync(meta, "http_5xx", {
+ status: pollStatus,
+ body: pollBody,
+ pollCount,
+ });
}
const parsed = pollResponseSchema.safeParse(pollBody);
if (!parsed.success) {
firePdfAsyncPollCount.observe(pollCount);
- return {
- kind: "fallback",
- result: await fallback("http_5xx", {
- error: String(parsed.error),
- body: pollBody,
- pollCount,
- }),
- };
+ failAsync(meta, "http_5xx", {
+ error: String(parsed.error),
+ body: pollBody,
+ pollCount,
+ });
}
if (TERMINAL_STATUSES.has(parsed.data.status)) {
@@ -148,17 +124,14 @@ export async function pollUntilTerminal(
: parsed.data.status === "expired"
? "terminal_expired"
: "terminal_cancelled";
- return {
- kind: "fallback",
- result: await fallback(reason, {
- status: parsed.data.status,
- errorClass: parsed.data.error_class,
- errorMessage: parsed.data.error_message,
- pollCount,
- }),
- };
+ failAsync(meta, reason, {
+ status: parsed.data.status,
+ errorClass: parsed.data.error_class,
+ errorMessage: parsed.data.error_message,
+ pollCount,
+ });
}
- return { kind: "done", poll: parsed.data, pollCount };
+ return { poll: parsed.data, pollCount };
}
lastDelay = nextPollDelay(lastDelay, parsed.data.retry_after_ms);
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts
index 633b19ecd..09a11a8c3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/result.ts
@@ -1,5 +1,4 @@
import type { Meta } from "../../..";
-import type { PDFProcessorResult } from "../types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import {
@@ -7,23 +6,18 @@ import {
resultResponseSchema,
type ResultResponse,
} from "./schema";
-import type { Fallback } from "./utils";
+import { failAsync } from "./utils";
-export type ResultDeps = {
+type ResultDeps = {
baseUrl: string;
scrapeId: string;
meta: Meta;
fetchImpl: typeof undiciFetch;
sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
- fallback: Fallback;
};
-export type ResultOutcome =
- | { kind: "ok"; result: ResultResponse }
- | { kind: "fallback"; result: PDFProcessorResult };
-
-export async function fetchResult(deps: ResultDeps): Promise<ResultOutcome> {
- const { baseUrl, scrapeId, meta, fetchImpl, sleep, fallback } = deps;
+export async function fetchResult(deps: ResultDeps): Promise<ResultResponse> {
+ const { baseUrl, scrapeId, meta, fetchImpl, sleep } = deps;
let retried409 = 0;
while (true) {
@@ -35,33 +29,24 @@ export async function fetchResult(deps: ResultDeps): Promise<ResultOutcome> {
});
} catch (error) {
if (error instanceof AbortManagerThrownError) throw error;
- return {
- kind: "fallback",
- result: await fallback("network_error", { error: String(error) }),
- };
+ failAsync(meta, "network_error", { error: String(error) });
}
const status = resp.status;
const body = await resp.json().catch(() => ({}));
if (status === 503) {
- return {
- kind: "fallback",
- result: await fallback("result_503", { body }),
- };
+ failAsync(meta, "result_503", { body });
}
if (status === 409) {
retried409++;
if (retried409 > 1) {
- return {
- kind: "fallback",
- result: await fallback("http_5xx", {
- status: 409,
- body,
- note: "result endpoint kept returning 409",
- }),
- };
+ failAsync(meta, "http_5xx", {
+ status: 409,
+ body,
+ note: "result endpoint kept returning 409",
+ });
}
meta.logger.info("FirePDF async result returned 409, re-polling once", {
scrapeId,
@@ -71,22 +56,16 @@ export async function fetchResult(deps: ResultDeps): Promise<ResultOutcome> {
}
if (status !== 200) {
- return {
- kind: "fallback",
- result: await fallback("http_5xx", { status, body }),
- };
+ failAsync(meta, "http_5xx", { status, body });
}
const parsed = resultResponseSchema.safeParse(body);
if (!parsed.success) {
- return {
- kind: "fallback",
- result: await fallback("http_5xx", {
- error: String(parsed.error),
- body,
- }),
- };
+ failAsync(meta, "http_5xx", {
+ error: String(parsed.error),
+ body,
+ });
}
- return { kind: "ok", result: parsed.data };
+ return parsed.data;
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index e239ed31c..220b6d769 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -1,22 +1,18 @@
import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
-import type { PDFProcessorResult } from "../types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
import { firePdfAsyncSubmittedTotal } from "./metrics";
import { submitResponseSchema } from "./schema";
-import type { Fallback } from "./utils";
+import { failAsync } from "./utils";
-export type SubmitOutcome =
- | {
- kind: "ok";
- lane: string | undefined;
- retryAfterMs: number | undefined;
- alreadyDone: boolean;
- }
- | { kind: "fallback"; result: PDFProcessorResult };
+type SubmitOutcome = {
+ lane: string | undefined;
+ retryAfterMs: number | undefined;
+ alreadyDone: boolean;
+};
-export type SubmitArgs = {
+type SubmitArgs = {
meta: Meta;
baseUrl: string;
base64Content: string;
@@ -25,7 +21,6 @@ export type SubmitArgs = {
mode: PDFMode | undefined;
deadlineAt: string;
fetchImpl: typeof undiciFetch;
- fallback: Fallback;
};
export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
@@ -38,7 +33,6 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
mode,
deadlineAt,
fetchImpl,
- fallback,
} = args;
const scrapeId = meta.id;
@@ -74,13 +68,13 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
json = await resp.json().catch(() => ({}));
} catch (error) {
if (error instanceof AbortManagerThrownError) throw error;
- return { kind: "fallback", result: await fallback("network_error", { error: String(error) }) };
+ failAsync(meta, "network_error", { error: String(error) });
}
- if (status === 404) return { kind: "fallback", result: await fallback("http_404") };
- if (status === 413) return { kind: "fallback", result: await fallback("http_413") };
- if (status === 429) return { kind: "fallback", result: await fallback("http_429") };
- if (status === 503) return { kind: "fallback", result: await fallback("http_503") };
+ if (status === 404) failAsync(meta, "http_404");
+ if (status === 413) failAsync(meta, "http_413");
+ if (status === 429) failAsync(meta, "http_429");
+ if (status === 503) failAsync(meta, "http_503");
if (status === 409) {
meta.logger.error("FirePDF async POST /jobs returned 409 scrape_id_conflict", {
@@ -101,22 +95,16 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
}
if (status !== 200 && status !== 202) {
- return {
- kind: "fallback",
- result: await fallback("http_5xx", { status, body: json }),
- };
+ failAsync(meta, "http_5xx", { status, body: json });
}
const parsed = submitResponseSchema.safeParse(json);
if (!parsed.success) {
- return {
- kind: "fallback",
- result: await fallback("http_5xx", {
- error: String(parsed.error),
- body: json,
- status,
- }),
- };
+ failAsync(meta, "http_5xx", {
+ error: String(parsed.error),
+ body: json,
+ status,
+ });
}
firePdfAsyncSubmittedTotal.labels(parsed.data.lane ?? "unknown").inc();
@@ -129,7 +117,6 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
});
return {
- kind: "ok",
lane: parsed.data.lane,
retryAfterMs: parsed.data.retry_after_ms,
alreadyDone: status === 200 && parsed.data.status === "done",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
index a96b75cab..f263c35d8 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -1,7 +1,4 @@
import type { Meta } from "../../..";
-import type { PDFMode } from "../../../../../controllers/v2/types";
-import type { PDFProcessorResult } from "../types";
-import type { scrapePDFWithFirePDF } from "../firePDF";
import {
MIN_DEADLINE_MS,
MAX_DEADLINE_MS,
@@ -59,31 +56,26 @@ export function computeDeadlineMs(scrapeTimeoutMs: number | undefined): number {
return Math.min(MAX_DEADLINE_MS, Math.max(MIN_DEADLINE_MS, candidate));
}
-export type Fallback = (
- reason: FallbackReason,
- extra?: Record<string, unknown>,
-) => Promise<PDFProcessorResult>;
-
-export type FallbackCtx = {
- meta: Meta;
- fallbackImpl: typeof scrapePDFWithFirePDF;
- base64Content: string;
- maxPages?: number;
- pagesProcessed?: number;
- mode?: PDFMode;
-};
+export class FirePdfAsyncFailure extends Error {
+ constructor(
+ public readonly reason: FallbackReason,
+ public readonly extra: Record<string, unknown> = {},
+ ) {
+ super(`fire-pdf async failed: ${reason}`);
+ this.name = "FirePdfAsyncFailure";
+ }
+}
-export function makeFallback(ctx: FallbackCtx): Fallback {
- const { meta, fallbackImpl, base64Content, maxPages, pagesProcessed, mode } =
- ctx;
- const scrapeId = meta.id;
- return async (reason, extra = {}) => {
- firePdfAsyncFallbackTotal.labels(reason).inc();
- meta.logger.warn("FirePDF async falling back to sync /ocr", {
- scrapeId,
- reason,
- ...extra,
- });
- return fallbackImpl(meta, base64Content, maxPages, pagesProcessed, mode);
- };
+export function failAsync(
+ meta: Meta,
+ reason: FallbackReason,
+ extra: Record<string, unknown> = {},
+): never {
+ firePdfAsyncFallbackTotal.labels(reason).inc();
+ meta.logger.warn("FirePDF async failed", {
+ scrapeId: meta.id,
+ reason,
+ ...extra,
+ });
+ throw new FirePdfAsyncFailure(reason, extra);
}