Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): switch image OCR on for the deployment with IMAGE_OCR_ENABLED (#4674)
Abimael Martell5d8b79d2acb1
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 5905d11f7..490c8b69a 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -159,3 +159,9 @@ X402_ENDPOINT_PRICE_USD=0.01# zero-data-retention requests are never exported.# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318# OTEL_SERVICE_NAME=firecrawl-api++# === Image OCR (optional) ===+# OCR raster image URLs and parse uploads (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP,+# WebP, AVIF) as one-page documents through FirePDF. Requires FIRE_PDF_BASE_URL.+# Off by default; when off, image URLs fail as unsupported files.+# IMAGE_OCR_ENABLED=truediff --git a/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.ts b/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.tsindex 9cc6e0f9f..ce779125a 100644--- a/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.ts@@ -16,29 +16,22 @@ import {} from "./lib";let identity: Identity;-let ungatedIdentity: Identity;beforeAll(async () => {- // Image OCR is rolled out per team through the imageOcr team flag.identity = await idmux({name: "scrape-image-ocr",concurrency: 100,credits: 1000000,- flags: { imageOcr: true },- });- ungatedIdentity = await idmux({- name: "scrape-image-ocr-ungated",- concurrency: 100,- credits: 1000000,});}, 10000 + scrapeTimeout);-// Image OCR rides on fire-engine (the browser hands the image bytes to the-// API) and FirePDF (which opens the bytes as a one-page scanned document).-// The flagged identity only exists where idmux is reachable: the self-hosted-// fallback identity carries no team flags.-const IMAGE_OCR_AVAILABLE = !!config.FIRE_PDF_BASE_URL && !!config.IDMUX_URL;-const SHOULD_RUN = !process.env.TEST_SUITE_SELF_HOSTED && IMAGE_OCR_AVAILABLE;+// Image OCR rides on FirePDF (which opens the bytes as a one-page scanned+// document) behind the IMAGE_OCR_ENABLED switch; image URLs additionally+// need fire-engine (the browser hands the image bytes to the API). The+// harness shares its environment with the API under test, so these mirror+// the server's decision.+const IMAGE_OCR_ON = !!config.FIRE_PDF_BASE_URL && config.IMAGE_OCR_ENABLED;+const SHOULD_RUN = !process.env.TEST_SUITE_SELF_HOSTED && IMAGE_OCR_ON;// 760x220 four-colour PNG rendered from three lines of text:// "Firecrawl OCR fixture"@@ -992,7 +985,7 @@ describeIf(SHOULD_RUN)("Image OCR (f-e and fire-pdf dependent)", () => {);});-describeIf(SHOULD_RUN)("Image OCR parse upload (fire-pdf dependent)", () => {+describeIf(IMAGE_OCR_ON)("Image OCR parse upload (fire-pdf dependent)", () => {it("OCRs an uploaded PNG",async () => {@@ -1141,7 +1134,9 @@ describeIf(SHOULD_RUN)("Image OCR parse upload (fire-pdf dependent)", () => {);});-describe("Image OCR for a team without the imageOcr flag", () => {+// With the switch off, images keep the historical unsupported-file rejection+// even when the request asks for the image parser.+describeIf(!IMAGE_OCR_ON)("Image OCR switched off", () => {it("rejects image uploads as unsupported",async () => {@@ -1156,7 +1151,7 @@ describe("Image OCR for a team without the imageOcr flag", () => {contentType: "image/png",},},- ungatedIdentity,+ identity,);expect(failure.code).toBe("UNSUPPORTED_FILE_TYPE");@@ -1176,7 +1171,7 @@ describe("Image OCR for a team without the imageOcr flag", () => {formats: ["markdown"],parsers: ["pdf", "image"],},- ungatedIdentity,+ identity,);expect(response.error).toContain("cannot process");diff --git a/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.tsindex 3a3cf0aa2..d1c845cc8 100644--- a/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts@@ -1,8 +1,10 @@import {ALLOW_TEST_SUITE_WEBSITE,describeIf,+ itIf,TEST_SUITE_WEBSITE,} from "../lib";+import { config } from "../../../config";import {idmux,Identity,@@ -21,14 +23,19 @@ beforeAll(async () => {});}, 10000 + scrapeTimeout);+// Mirrors the API's image OCR switch (the harness shares its environment+// with the server under test).+const IMAGE_OCR_ON = !!config.FIRE_PDF_BASE_URL && config.IMAGE_OCR_ENABLED;+// The browser engine hands files it cannot render back to the API, which// routes them to a parser by content type β and servers mislabel. Static// hosts serve a `.jp2` file as image/jp2 whatever its bytes are, so// mislabeled-pdf.jp2 reproduces a PDF served with an image content type,-// while tiny-image.jp2 is a real JPEG 2000 image. This identity has no-// imageOcr team flag, so the real image must keep the historical rejection-// (the flagged behaviour lives in scrape-image-ocr.test.ts). Only-// fire-engine performs the handoff, hence the gate.+// while tiny-image.jp2 is a real JPEG 2000 image, which follows the image+// OCR switch: rejected while it is off, an empty document (it is 16 px and+// has no text) while it is on. The OCR behaviour itself lives in+// scrape-image-ocr.test.ts. Only fire-engine performs the handoff, hence+// the gate.describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)("Mislabeled file handoff (f-e dependent)",() => {@@ -51,8 +58,8 @@ describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)(scrapeTimeout,);- it(- "keeps rejecting a real JPEG 2000 image for a team without image OCR",+ itIf(!IMAGE_OCR_ON)(+ "keeps rejecting a real JPEG 2000 image while image OCR is off",async () => {const response = await scrapeWithFailure({@@ -67,5 +74,23 @@ describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)(},scrapeTimeout,);++ itIf(IMAGE_OCR_ON)(+ "OCRs a real JPEG 2000 image while image OCR is on",+ async () => {+ const response = await scrape(+ {+ url: `${TEST_SUITE_WEBSITE}/tiny-image.jp2`,+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.metadata.contentType).toBe("image/jp2");+ expect(response.metadata.statusCode).toBe(200);+ expect(response.markdown).toBe("");+ },+ scrapeTimeout,+ );},);diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 3feeacffc..44b9f4304 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -383,6 +383,9 @@ const configSchema = z.object({FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),FIRE_PDF_BASE_URL: z.string().optional(),FIRE_PDF_API_KEY: z.string().optional(),+ // Raster image OCR of image URLs and parse uploads through FirePDF (see+ // lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.+ IMAGE_OCR_ENABLED: z.stringbool().default(false),// Async /jobs rollout is a separate, server-controlled cohort inside// traffic already selected for FirePDF. It is disabled by default.FIRE_PDF_ASYNC_PERCENT: z.coerce.number().min(0).max(100).default(0),diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 1954b9ce4..ec34c63eb 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1397,9 +1397,6 @@ export type TeamFlags = {>;// routes the team's new queue work to the FoundationDB backendnuqFdb?: boolean;- // enables OCR of raster image URLs and uploads through FirePDF (see- // lib/image-ocr-gate.ts); rolled out per team- imageOcr?: boolean;/*** Per-endpoint rate-limit overrides, in requests per minute. A value here* replaces the computed limit for that mode, so the Autumn multiplier isdiff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.tsindex f971ee99e..627d56313 100644--- a/apps/api/src/controllers/v2/parse-upload.ts+++ b/apps/api/src/controllers/v2/parse-upload.ts@@ -293,7 +293,7 @@ export async function parseUploadUrlController(}const { filename, contentType, declaredSizeBytes } = parsed.data;- const imageOcrEnabled = isImageOcrEnabled(req.acuc?.flags);+ const imageOcrEnabled = isImageOcrEnabled();if (!isSupportedParseUpload(filename, contentType, imageOcrEnabled)) {return res.status(400).json({success: false,@@ -531,8 +531,8 @@ async function resolveUploadRef();if (!kind) {// The type was accepted when the upload URL was minted, so this only- // happens when eligibility changed in between (e.g. the imageOcr team- // flag was turned off). Discard the upload instead of stranding it and+ // happens when eligibility changed in between (e.g. image OCR was+ // switched off). Discard the upload instead of stranding it and// its quota reservation.localUploads.delete(payload.uploadId);await releaseRejectedUploadRef(payload);@@ -648,10 +648,7 @@ export async function parseUploadRefPayloadMiddleware(}try {- const resolved = await resolveUploadRef(- payload,- isImageOcrEnabled(req.acuc?.flags),- );+ const resolved = await resolveUploadRef(payload, isImageOcrEnabled());const { uploadRef: _uploadRef, ...options } = req.body;req.body = {...options,diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex f54a0ebfc..709df99a9 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -84,8 +84,8 @@ export function detectUploadedFileKind(return "document";}- // Image uploads are OCR'd through FirePDF for teams with the imageOcr- // flag; for everyone else they stay unsupported.+ // Image uploads are OCR'd through FirePDF where the deployment has image+ // OCR on (lib/image-ocr-gate.ts); otherwise they stay unsupported.const isImage =imageOcrEnabled &&(IMAGE_EXTENSIONS.has(extension) ||@@ -258,11 +258,7 @@ export function parseMultipartPayloadMiddleware(}}- // authMiddleware runs before this middleware, so the team's flags are- // available to decide whether image uploads are accepted.- const imageOcrEnabled = isImageOcrEnabled(- (req as unknown as RequestWithAuth).acuc?.flags,- );+ const imageOcrEnabled = isImageOcrEnabled();const kind = detectUploadedFileKind(file.originalname || "",file.mimetype,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 9cd769e9c..05cf16dc5 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -527,9 +527,9 @@ const pdfParserWithOptions = z/*** Raster image OCR (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP). Like `pdf` it is* part of the default list, so a request that says nothing about parsers OCRs- * image URLs (behind the imageOcr team flag while it rolls out); an explicit- * list that omits it (`["pdf"]`, `[]`) opts out and keeps the historical- * unsupported-file rejection. The object form carries no options yet; it+ * image URLs (where the deployment has image OCR on, see+ * lib/image-ocr-gate.ts); an explicit list that omits it (`["pdf"]`, `[]`)+ * opts out and keeps the historical unsupported-file rejection. The object form carries no options yet; it* exists so options can be added later without a breaking change.*/const imageParserWithOptions = z.strictObject({diff --git a/apps/api/src/lib/image-ocr-gate.test.ts b/apps/api/src/lib/image-ocr-gate.test.tsindex 3d0ebfef6..980164047 100644--- a/apps/api/src/lib/image-ocr-gate.test.ts+++ b/apps/api/src/lib/image-ocr-gate.test.ts@@ -1,85 +1,59 @@-import { beforeEach, describe, expect, it, vi } from "vitest";+import { describe, expect, it, vi } from "vitest";import { config } from "../config";-import { getACUCTeam } from "../controllers/auth";import { imageOcrGate, isImageOcrEnabled } from "./image-ocr-gate";vi.mock("../config", () => ({config: {FIRE_PDF_BASE_URL: "http://fire-pdf.test",+ IMAGE_OCR_ENABLED: true,},}));-vi.mock("../controllers/auth", () => ({- getACUCTeam: vi.fn(),-}));--vi.mock("./logger", () => ({- logger: { warn: vi.fn() },-}));--const mockedGetACUCTeam = vi.mocked(getACUCTeam);+const mutableConfig = config as {+ FIRE_PDF_BASE_URL?: string;+ IMAGE_OCR_ENABLED: boolean;+};++function withConfig<T>(+ overrides: Partial<typeof mutableConfig>,+ run: () => T,+): T {+ const previous = { ...mutableConfig };+ Object.assign(mutableConfig, overrides);+ try {+ return run();+ } finally {+ Object.assign(mutableConfig, previous);+ }+}describe("isImageOcrEnabled", () => {- it("requires the imageOcr team flag", () => {- expect(isImageOcrEnabled(null)).toBe(false);- expect(isImageOcrEnabled(undefined)).toBe(false);- expect(isImageOcrEnabled({})).toBe(false);- expect(isImageOcrEnabled({ imageOcr: false })).toBe(false);- expect(isImageOcrEnabled({ imageOcr: true })).toBe(true);+ it("follows the deployment switch", () => {+ expect(isImageOcrEnabled()).toBe(true);+ withConfig({ IMAGE_OCR_ENABLED: false }, () => {+ expect(isImageOcrEnabled()).toBe(false);+ });});- it("requires FirePDF to be configured even for flagged teams", () => {- const mutable = config as { FIRE_PDF_BASE_URL?: string };- const previous = mutable.FIRE_PDF_BASE_URL;- mutable.FIRE_PDF_BASE_URL = undefined;- try {- expect(isImageOcrEnabled({ imageOcr: true })).toBe(false);- } finally {- mutable.FIRE_PDF_BASE_URL = previous;- }+ it("requires FirePDF to be configured even with the switch on", () => {+ withConfig({ FIRE_PDF_BASE_URL: undefined }, () => {+ expect(isImageOcrEnabled()).toBe(false);+ });});});describe("imageOcrGate", () => {- beforeEach(() => {- mockedGetACUCTeam.mockReset();- });-- it("is off without the image parser and never looks the team up", async () => {- await expect(- imageOcrGate("team", { imageOcr: true }, false)(),- ).resolves.toBe(false);- await expect(imageOcrGate("team", undefined, false)()).resolves.toBe(false);- expect(mockedGetACUCTeam).not.toHaveBeenCalled();+ it("is off when the request did not ask for the image parser", async () => {+ await expect(imageOcrGate(false)()).resolves.toBe(false);});- it("uses the flags carried on the job without a lookup", async () => {- await expect(- imageOcrGate("team", { imageOcr: true }, true)(),- ).resolves.toBe(true);- await expect(imageOcrGate("team", null, true)()).resolves.toBe(false);- expect(mockedGetACUCTeam).not.toHaveBeenCalled();- });-- it("falls back to the cached team ACUC once per scrape when the job carries no flags", async () => {- mockedGetACUCTeam.mockResolvedValueOnce({- flags: { imageOcr: true },- } as Awaited<ReturnType<typeof getACUCTeam>>);- const gate = imageOcrGate("team", undefined, true);- await expect(gate()).resolves.toBe(true);- await expect(gate()).resolves.toBe(true);- expect(mockedGetACUCTeam).toHaveBeenCalledTimes(1);- expect(mockedGetACUCTeam).toHaveBeenCalledWith("team");-- mockedGetACUCTeam.mockResolvedValueOnce(null);- await expect(imageOcrGate("team", undefined, true)()).resolves.toBe(false);- });-- it("leaves image OCR off when the lookup fails or there is no team", async () => {- mockedGetACUCTeam.mockRejectedValueOnce(new Error("redis down"));- await expect(imageOcrGate("team", undefined, true)()).resolves.toBe(false);- await expect(imageOcrGate(undefined, undefined, true)()).resolves.toBe(- false,+ it("follows the deployment switch for a request that asked", async () => {+ await expect(imageOcrGate(true)()).resolves.toBe(true);+ await withConfig({ IMAGE_OCR_ENABLED: false }, () =>+ expect(imageOcrGate(true)()).resolves.toBe(false),+ );+ await withConfig({ FIRE_PDF_BASE_URL: undefined }, () =>+ expect(imageOcrGate(true)()).resolves.toBe(false),);});});diff --git a/apps/api/src/lib/image-ocr-gate.ts b/apps/api/src/lib/image-ocr-gate.tsindex d2c2a84f0..213352789 100644--- a/apps/api/src/lib/image-ocr-gate.ts+++ b/apps/api/src/lib/image-ocr-gate.ts@@ -1,70 +1,31 @@import { config } from "../config";-import type { TeamFlags } from "../controllers/v1/types";-import { getACUCTeam } from "../controllers/auth";-import { logger } from "./logger";/**- * Raster image OCR rides on FirePDF and is rolled out per team through the- * `imageOcr` team flag. Every entry point (URL-extension routing, the browser- * handoff, parse uploads) consults this one check, so a team without the flag- * gets exactly the pre-existing behaviour.+ * Raster image OCR rides on FirePDF and is switched on for the whole+ * deployment by `IMAGE_OCR_ENABLED`. Every entry point (URL-extension+ * routing, the browser handoff, parse uploads) consults this one check, so+ * with the switch off every request gets exactly the pre-existing+ * unsupported-file behaviour.*/-export function isImageOcrEnabled(- teamFlags: TeamFlags | null | undefined,-): boolean {- return !!config.FIRE_PDF_BASE_URL && teamFlags?.imageOcr === true;+export function isImageOcrEnabled(): boolean {+ return !!config.FIRE_PDF_BASE_URL && config.IMAGE_OCR_ENABLED;}-/** Per-scrape gate: resolved lazily on first call and memoized. */+/**+ * Per-scrape gate. Async so the callers that consult it late (the browser+ * handoff sniff, the image engine, the index) share one shape even though+ * the answer is known when the scrape starts.+ */export type ImageOcrGate = () => Promise<boolean>;-const OFF: Promise<boolean> = Promise.resolve(false);-/*** Builds the per-scrape gate: whether this request may OCR raster images.** Two conditions fold into it. The request's `parsers` must include the* `image` parser β it does by default, and a parse upload of an image counts- * regardless β and the team must have the `imageOcr` flag while the feature- * rolls out. A request that opted out is settled up front without any I/O.- *- * For the team side, single scrapes and parse uploads carry the- * authenticated team's flags in their internalOptions and resolve without- * I/O; batch-scrape and crawl jobs do not, so for those the flags come from- * the cached team ACUC. That lookup is deferred until a caller actually needs- * the answer (an image-extension URL, an image handoff, the image engine, a- * cached image document) and memoized, so the ordinary HTML documents that- * make up almost every crawl never pay for it. Any lookup failure keeps the- * pre-existing behaviour.+ * regardless β and the deployment switch must be on (see isImageOcrEnabled).*/-export function imageOcrGate(- teamId: string | undefined,- teamFlags: TeamFlags | null | undefined,- requested: boolean,-): ImageOcrGate {- if (!requested) return () => OFF;- let pending: Promise<boolean> | undefined;- return () => {- pending ??= resolveImageOcrEnabled(teamId, teamFlags);- return pending;- };-}--async function resolveImageOcrEnabled(- teamId: string | undefined,- teamFlags: TeamFlags | null | undefined,-): Promise<boolean> {- if (!config.FIRE_PDF_BASE_URL) return false;- if (teamFlags !== undefined) return isImageOcrEnabled(teamFlags);- if (!teamId) return false;- try {- const acuc = await getACUCTeam(teamId);- return isImageOcrEnabled(acuc?.flags ?? null);- } catch (error) {- logger.warn("Failed to resolve team flags for image OCR; leaving it off", {- teamId,- error,- });- return false;- }+export function imageOcrGate(requested: boolean): ImageOcrGate {+ const enabled = Promise.resolve(requested && isImageOcrEnabled());+ return () => enabled;}diff --git a/apps/api/src/scraper/scrapeURL/engines/image/index.ts b/apps/api/src/scraper/scrapeURL/engines/image/index.tsindex 7623bfb1f..9ad468510 100644--- a/apps/api/src/scraper/scrapeURL/engines/image/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/image/index.ts@@ -17,9 +17,10 @@ import {* (PNG/JPEG/JPEG 2000/TIFF/GIF/BMP), finds no text layer, and runs the same* layout + OCR path a scanned PDF page takes.*- * OCR is on by default and opt-out per request (a `parsers` list without- * `image`; a parse upload of an image always counts), and rolled out per- * team (imageOcr flag); both are folded into `meta.imageOcrEnabled`. The pdf+ * Image OCR is off until the deployment enables it (`IMAGE_OCR_ENABLED`,+ * default false). Once it is on, every request gets it unless it opts out+ * per request (a `parsers` list without `image`; a parse upload of an image+ * always counts); both conditions are folded into `meta.imageOcrEnabled`. The pdf* parser's options (mode, maxPages, pages, blocks, pageMarkers) are not* consulted: an image has no text layer to fall back on, so every admitted* image is OCR'd. A caller who wants the bytes instead uses the `rawBase64`@@ -58,10 +59,10 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {}// Requests already identified as images (a browser handoff, the image- // flag, a forceEngine pin, or an image-extension URL) resolve the- // per-team gate up front. Anything else is a waterfall tail that has to- // sniff its bytes first, so an ordinary failed page never pays for a- // team lookup here.+ // flag, a forceEngine pin, or an image-extension URL) consult the gate+ // up front. Anything else is a waterfall tail that has to sniff its+ // bytes first, so an ordinary failed page is rejected on its bytes, not+ // on the gate.const knownImage =meta.imagePrefetch != null ||meta.featureFlags.has("image") ||@@ -70,7 +71,7 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {new URL(meta.rewrittenUrl ?? meta.url).pathname,) !== null;if (knownImage && !(await meta.imageOcrEnabled())) {- // Opted out (parsers without image) or no team flag: the request gets+ // Opted out (parsers without image) or switched off: the request gets// the unsupported-file error the URL path has always produced, whose// message names the parser.throw new UnsupportedFileError(@@ -132,7 +133,7 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {}// A tail request whose bytes turned out to be an image: consult the gate- // now that the lookup is warranted.+ // now that it is warranted.if (!knownImage && !(await meta.imageOcrEnabled())) {throw new UnsupportedFileError(contentType);}diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex c9ac123f3..c44ef7f77 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -97,8 +97,8 @@ const engines: Engine[] = ["pdf","document",// Image OCR needs FirePDF; without it the engine would only be a wasted- // tail download on every failed scrape. Per-team enablement is decided- // where images are routed (the imageOcr team flag), not here.+ // tail download on every failed scrape. Whether images are actually OCR'd+ // is decided where they are routed (lib/image-ocr-gate.ts), not here....(config.FIRE_PDF_BASE_URL ? ["image" as const] : []),];diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 586a55b17..08649ec09 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -118,7 +118,7 @@ async function feResToImagePrefetch(* Sniffs a browser handoff for a raster image the image engine can OCR.* Returns the canonical content type from the magic bytes, or null when the* payload is missing, is not a supported format, or the request may not OCR- * images (no image parser, or a team without the flag) β in which case the+ * images (no image parser, or the switch is off) β in which case the* caller falls through to the historical unsupported-file rejection.*/async function sniffImageHandoff(@@ -129,8 +129,8 @@ async function sniffImageHandoff(if (content === undefined) return null;const contentType = sniffImageContentTypeFromBase64(content);if (contentType === null) return null;- // Only now consult the gate (image parser + team flag): this is the one- // lookup an image request may cost, and non-image responses never reach it.+ // Only now consult the gate (image parser + deployment switch); non-image+ // responses never reach it.return (await imageOcrEnabled()) ? contentType : null;}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 91d9b3022..c9649e6c2 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -170,10 +170,8 @@ export type Meta = {mock: MockState | null;/** Whether this scrape may OCR raster images: the request's parsers* include `image` (the default; a parse upload of an image always counts)- * and the team has the imageOcr flag with FirePDF configured. Lazy and- * memoized: the browser handoff, the image engine and the index only ask- * once a request actually looks like an image, so plain documents never- * pay for the team lookup. */+ * and the deployment has image OCR switched on with FirePDF configured.+ * Consulted by the browser handoff, the image engine and the index. */imageOcrEnabled: ImageOcrGate;pdfPrefetch:| {@@ -346,9 +344,9 @@ function buildFeatureFlags(flags.add("pdf");} else if (imageExtensionFromUrlPath(lowerPath) !== null && imageOcrEnabled) {// Raster images are OCR'd through FirePDF when the request's parsers- // include `image` (the default) and the team has the imageOcr flag (see- // engines/image). Everyone else stays on the ordinary waterfall and- // fails as an unsupported file, exactly as before.+ // include `image` (the default) and the deployment has image OCR on (see+ // engines/image and lib/image-ocr-gate). Everything else stays on the+ // ordinary waterfall and fails as an unsupported file, exactly as before.flags.add("image");}@@ -538,16 +536,14 @@ async function buildMetaObject(const effectiveOptions = applyScrapeOptionsDefaults(options);// Image OCR follows the parsers option: on by default, off when the caller// sends a list without `image`. A parse upload of an image is a request to- // parse that file, so it counts regardless. The team flag is checked lazily+ // parse that file, so it counts regardless. The deployment switch sits// behind this.const imageOcrEnabled = imageOcrGate(- internalOptions.teamId,- internalOptions.teamFlags,shouldParseImages(effectiveOptions.parsers) ||internalOptions.uploadedFile?.kind === "image",);// Only an image-extension URL needs the answer up front; everything else- // resolves lazily on an image handoff, if one ever happens.+ // consults the gate on an image handoff, if one ever happens.const imageOcrForUrl =imageExtensionFromUrlPath(new URL(url).pathname) !== null &&(await imageOcrEnabled());