Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): switch image OCR on for the deployment with IMAGE_OCR_ENABLED (#4674)

Abimael Martell5d8b79d2acb1
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 5905d11f7..490c8b69a 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -159,3 +159,9 @@ X402_ENDPOINT_PRICE_USD=0.01
# zero-data-retention requests are never exported.
# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
# OTEL_SERVICE_NAME=firecrawl-api
+
+# === Image OCR (optional) ===
+# OCR raster image URLs and parse uploads (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP,
+# WebP, AVIF) as one-page documents through FirePDF. Requires FIRE_PDF_BASE_URL.
+# Off by default; when off, image URLs fail as unsupported files.
+# IMAGE_OCR_ENABLED=true
diff --git a/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.ts b/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.ts
index 9cc6e0f9f..ce779125a 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-image-ocr.test.ts
@@ -16,29 +16,22 @@ import {
} from "./lib";
let identity: Identity;
-let ungatedIdentity: Identity;
beforeAll(async () => {
- // Image OCR is rolled out per team through the imageOcr team flag.
identity = await idmux({
name: "scrape-image-ocr",
concurrency: 100,
credits: 1000000,
- flags: { imageOcr: true },
- });
- ungatedIdentity = await idmux({
- name: "scrape-image-ocr-ungated",
- concurrency: 100,
- credits: 1000000,
});
}, 10000 + scrapeTimeout);
-// Image OCR rides on fire-engine (the browser hands the image bytes to the
-// API) and FirePDF (which opens the bytes as a one-page scanned document).
-// The flagged identity only exists where idmux is reachable: the self-hosted
-// fallback identity carries no team flags.
-const IMAGE_OCR_AVAILABLE = !!config.FIRE_PDF_BASE_URL && !!config.IDMUX_URL;
-const SHOULD_RUN = !process.env.TEST_SUITE_SELF_HOSTED && IMAGE_OCR_AVAILABLE;
+// Image OCR rides on FirePDF (which opens the bytes as a one-page scanned
+// document) behind the IMAGE_OCR_ENABLED switch; image URLs additionally
+// need fire-engine (the browser hands the image bytes to the API). The
+// harness shares its environment with the API under test, so these mirror
+// the server's decision.
+const IMAGE_OCR_ON = !!config.FIRE_PDF_BASE_URL && config.IMAGE_OCR_ENABLED;
+const SHOULD_RUN = !process.env.TEST_SUITE_SELF_HOSTED && IMAGE_OCR_ON;
// 760x220 four-colour PNG rendered from three lines of text:
// "Firecrawl OCR fixture"
@@ -992,7 +985,7 @@ describeIf(SHOULD_RUN)("Image OCR (f-e and fire-pdf dependent)", () => {
);
});
-describeIf(SHOULD_RUN)("Image OCR parse upload (fire-pdf dependent)", () => {
+describeIf(IMAGE_OCR_ON)("Image OCR parse upload (fire-pdf dependent)", () => {
it(
"OCRs an uploaded PNG",
async () => {
@@ -1141,7 +1134,9 @@ describeIf(SHOULD_RUN)("Image OCR parse upload (fire-pdf dependent)", () => {
);
});
-describe("Image OCR for a team without the imageOcr flag", () => {
+// With the switch off, images keep the historical unsupported-file rejection
+// even when the request asks for the image parser.
+describeIf(!IMAGE_OCR_ON)("Image OCR switched off", () => {
it(
"rejects image uploads as unsupported",
async () => {
@@ -1156,7 +1151,7 @@ describe("Image OCR for a team without the imageOcr flag", () => {
contentType: "image/png",
},
},
- ungatedIdentity,
+ identity,
);
expect(failure.code).toBe("UNSUPPORTED_FILE_TYPE");
@@ -1176,7 +1171,7 @@ describe("Image OCR for a team without the imageOcr flag", () => {
formats: ["markdown"],
parsers: ["pdf", "image"],
},
- ungatedIdentity,
+ identity,
);
expect(response.error).toContain("cannot process");
diff --git a/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts
index 3a3cf0aa2..d1c845cc8 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-mislabeled-files.test.ts
@@ -1,8 +1,10 @@
import {
ALLOW_TEST_SUITE_WEBSITE,
describeIf,
+ itIf,
TEST_SUITE_WEBSITE,
} from "../lib";
+import { config } from "../../../config";
import {
idmux,
Identity,
@@ -21,14 +23,19 @@ beforeAll(async () => {
});
}, 10000 + scrapeTimeout);
+// Mirrors the API's image OCR switch (the harness shares its environment
+// with the server under test).
+const IMAGE_OCR_ON = !!config.FIRE_PDF_BASE_URL && config.IMAGE_OCR_ENABLED;
+
// The browser engine hands files it cannot render back to the API, which
// routes them to a parser by content type β€” and servers mislabel. Static
// hosts serve a `.jp2` file as image/jp2 whatever its bytes are, so
// mislabeled-pdf.jp2 reproduces a PDF served with an image content type,
-// while tiny-image.jp2 is a real JPEG 2000 image. This identity has no
-// imageOcr team flag, so the real image must keep the historical rejection
-// (the flagged behaviour lives in scrape-image-ocr.test.ts). Only
-// fire-engine performs the handoff, hence the gate.
+// while tiny-image.jp2 is a real JPEG 2000 image, which follows the image
+// OCR switch: rejected while it is off, an empty document (it is 16 px and
+// has no text) while it is on. The OCR behaviour itself lives in
+// scrape-image-ocr.test.ts. Only fire-engine performs the handoff, hence
+// the gate.
describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)(
"Mislabeled file handoff (f-e dependent)",
() => {
@@ -51,8 +58,8 @@ describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)(
scrapeTimeout,
);
- it(
- "keeps rejecting a real JPEG 2000 image for a team without image OCR",
+ itIf(!IMAGE_OCR_ON)(
+ "keeps rejecting a real JPEG 2000 image while image OCR is off",
async () => {
const response = await scrapeWithFailure(
{
@@ -67,5 +74,23 @@ describeIf(!process.env.TEST_SUITE_SELF_HOSTED && ALLOW_TEST_SUITE_WEBSITE)(
},
scrapeTimeout,
);
+
+ itIf(IMAGE_OCR_ON)(
+ "OCRs a real JPEG 2000 image while image OCR is on",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${TEST_SUITE_WEBSITE}/tiny-image.jp2`,
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.metadata.contentType).toBe("image/jp2");
+ expect(response.metadata.statusCode).toBe(200);
+ expect(response.markdown).toBe("");
+ },
+ scrapeTimeout,
+ );
},
);
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 3feeacffc..44b9f4304 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -383,6 +383,9 @@ const configSchema = z.object({
FIRE_PDF_PERCENT: z.coerce.number().min(0).max(100).default(10),
FIRE_PDF_BASE_URL: z.string().optional(),
FIRE_PDF_API_KEY: z.string().optional(),
+ // Raster image OCR of image URLs and parse uploads through FirePDF (see
+ // lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.
+ IMAGE_OCR_ENABLED: z.stringbool().default(false),
// Async /jobs rollout is a separate, server-controlled cohort inside
// traffic already selected for FirePDF. It is disabled by default.
FIRE_PDF_ASYNC_PERCENT: z.coerce.number().min(0).max(100).default(0),
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 1954b9ce4..ec34c63eb 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1397,9 +1397,6 @@ export type TeamFlags = {
>;
// routes the team's new queue work to the FoundationDB backend
nuqFdb?: boolean;
- // enables OCR of raster image URLs and uploads through FirePDF (see
- // lib/image-ocr-gate.ts); rolled out per team
- imageOcr?: boolean;
/**
* Per-endpoint rate-limit overrides, in requests per minute. A value here
* replaces the computed limit for that mode, so the Autumn multiplier is
diff --git a/apps/api/src/controllers/v2/parse-upload.ts b/apps/api/src/controllers/v2/parse-upload.ts
index f971ee99e..627d56313 100644
--- a/apps/api/src/controllers/v2/parse-upload.ts
+++ b/apps/api/src/controllers/v2/parse-upload.ts
@@ -293,7 +293,7 @@ export async function parseUploadUrlController(
}
const { filename, contentType, declaredSizeBytes } = parsed.data;
- const imageOcrEnabled = isImageOcrEnabled(req.acuc?.flags);
+ const imageOcrEnabled = isImageOcrEnabled();
if (!isSupportedParseUpload(filename, contentType, imageOcrEnabled)) {
return res.status(400).json({
success: false,
@@ -531,8 +531,8 @@ async function resolveUploadRef(
);
if (!kind) {
// The type was accepted when the upload URL was minted, so this only
- // happens when eligibility changed in between (e.g. the imageOcr team
- // flag was turned off). Discard the upload instead of stranding it and
+ // happens when eligibility changed in between (e.g. image OCR was
+ // switched off). Discard the upload instead of stranding it and
// its quota reservation.
localUploads.delete(payload.uploadId);
await releaseRejectedUploadRef(payload);
@@ -648,10 +648,7 @@ export async function parseUploadRefPayloadMiddleware(
}
try {
- const resolved = await resolveUploadRef(
- payload,
- isImageOcrEnabled(req.acuc?.flags),
- );
+ const resolved = await resolveUploadRef(payload, isImageOcrEnabled());
const { uploadRef: _uploadRef, ...options } = req.body;
req.body = {
...options,
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index f54a0ebfc..709df99a9 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -84,8 +84,8 @@ export function detectUploadedFileKind(
return "document";
}
- // Image uploads are OCR'd through FirePDF for teams with the imageOcr
- // flag; for everyone else they stay unsupported.
+ // Image uploads are OCR'd through FirePDF where the deployment has image
+ // OCR on (lib/image-ocr-gate.ts); otherwise they stay unsupported.
const isImage =
imageOcrEnabled &&
(IMAGE_EXTENSIONS.has(extension) ||
@@ -258,11 +258,7 @@ export function parseMultipartPayloadMiddleware(
}
}
- // authMiddleware runs before this middleware, so the team's flags are
- // available to decide whether image uploads are accepted.
- const imageOcrEnabled = isImageOcrEnabled(
- (req as unknown as RequestWithAuth).acuc?.flags,
- );
+ const imageOcrEnabled = isImageOcrEnabled();
const kind = detectUploadedFileKind(
file.originalname || "",
file.mimetype,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 9cd769e9c..05cf16dc5 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -527,9 +527,9 @@ const pdfParserWithOptions = z
/**
* Raster image OCR (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP). Like `pdf` it is
* part of the default list, so a request that says nothing about parsers OCRs
- * image URLs (behind the imageOcr team flag while it rolls out); an explicit
- * list that omits it (`["pdf"]`, `[]`) opts out and keeps the historical
- * unsupported-file rejection. The object form carries no options yet; it
+ * image URLs (where the deployment has image OCR on, see
+ * lib/image-ocr-gate.ts); an explicit list that omits it (`["pdf"]`, `[]`)
+ * opts out and keeps the historical unsupported-file rejection. The object form carries no options yet; it
* exists so options can be added later without a breaking change.
*/
const imageParserWithOptions = z.strictObject({
diff --git a/apps/api/src/lib/image-ocr-gate.test.ts b/apps/api/src/lib/image-ocr-gate.test.ts
index 3d0ebfef6..980164047 100644
--- a/apps/api/src/lib/image-ocr-gate.test.ts
+++ b/apps/api/src/lib/image-ocr-gate.test.ts
@@ -1,85 +1,59 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { describe, expect, it, vi } from "vitest";
import { config } from "../config";
-import { getACUCTeam } from "../controllers/auth";
import { imageOcrGate, isImageOcrEnabled } from "./image-ocr-gate";
vi.mock("../config", () => ({
config: {
FIRE_PDF_BASE_URL: "http://fire-pdf.test",
+ IMAGE_OCR_ENABLED: true,
},
}));
-vi.mock("../controllers/auth", () => ({
- getACUCTeam: vi.fn(),
-}));
-
-vi.mock("./logger", () => ({
- logger: { warn: vi.fn() },
-}));
-
-const mockedGetACUCTeam = vi.mocked(getACUCTeam);
+const mutableConfig = config as {
+ FIRE_PDF_BASE_URL?: string;
+ IMAGE_OCR_ENABLED: boolean;
+};
+
+function withConfig<T>(
+ overrides: Partial<typeof mutableConfig>,
+ run: () => T,
+): T {
+ const previous = { ...mutableConfig };
+ Object.assign(mutableConfig, overrides);
+ try {
+ return run();
+ } finally {
+ Object.assign(mutableConfig, previous);
+ }
+}
describe("isImageOcrEnabled", () => {
- it("requires the imageOcr team flag", () => {
- expect(isImageOcrEnabled(null)).toBe(false);
- expect(isImageOcrEnabled(undefined)).toBe(false);
- expect(isImageOcrEnabled({})).toBe(false);
- expect(isImageOcrEnabled({ imageOcr: false })).toBe(false);
- expect(isImageOcrEnabled({ imageOcr: true })).toBe(true);
+ it("follows the deployment switch", () => {
+ expect(isImageOcrEnabled()).toBe(true);
+ withConfig({ IMAGE_OCR_ENABLED: false }, () => {
+ expect(isImageOcrEnabled()).toBe(false);
+ });
});
- it("requires FirePDF to be configured even for flagged teams", () => {
- const mutable = config as { FIRE_PDF_BASE_URL?: string };
- const previous = mutable.FIRE_PDF_BASE_URL;
- mutable.FIRE_PDF_BASE_URL = undefined;
- try {
- expect(isImageOcrEnabled({ imageOcr: true })).toBe(false);
- } finally {
- mutable.FIRE_PDF_BASE_URL = previous;
- }
+ it("requires FirePDF to be configured even with the switch on", () => {
+ withConfig({ FIRE_PDF_BASE_URL: undefined }, () => {
+ expect(isImageOcrEnabled()).toBe(false);
+ });
});
});
describe("imageOcrGate", () => {
- beforeEach(() => {
- mockedGetACUCTeam.mockReset();
- });
-
- it("is off without the image parser and never looks the team up", async () => {
- await expect(
- imageOcrGate("team", { imageOcr: true }, false)(),
- ).resolves.toBe(false);
- await expect(imageOcrGate("team", undefined, false)()).resolves.toBe(false);
- expect(mockedGetACUCTeam).not.toHaveBeenCalled();
+ it("is off when the request did not ask for the image parser", async () => {
+ await expect(imageOcrGate(false)()).resolves.toBe(false);
});
- it("uses the flags carried on the job without a lookup", async () => {
- await expect(
- imageOcrGate("team", { imageOcr: true }, true)(),
- ).resolves.toBe(true);
- await expect(imageOcrGate("team", null, true)()).resolves.toBe(false);
- expect(mockedGetACUCTeam).not.toHaveBeenCalled();
- });
-
- it("falls back to the cached team ACUC once per scrape when the job carries no flags", async () => {
- mockedGetACUCTeam.mockResolvedValueOnce({
- flags: { imageOcr: true },
- } as Awaited<ReturnType<typeof getACUCTeam>>);
- const gate = imageOcrGate("team", undefined, true);
- await expect(gate()).resolves.toBe(true);
- await expect(gate()).resolves.toBe(true);
- expect(mockedGetACUCTeam).toHaveBeenCalledTimes(1);
- expect(mockedGetACUCTeam).toHaveBeenCalledWith("team");
-
- mockedGetACUCTeam.mockResolvedValueOnce(null);
- await expect(imageOcrGate("team", undefined, true)()).resolves.toBe(false);
- });
-
- it("leaves image OCR off when the lookup fails or there is no team", async () => {
- mockedGetACUCTeam.mockRejectedValueOnce(new Error("redis down"));
- await expect(imageOcrGate("team", undefined, true)()).resolves.toBe(false);
- await expect(imageOcrGate(undefined, undefined, true)()).resolves.toBe(
- false,
+ it("follows the deployment switch for a request that asked", async () => {
+ await expect(imageOcrGate(true)()).resolves.toBe(true);
+ await withConfig({ IMAGE_OCR_ENABLED: false }, () =>
+ expect(imageOcrGate(true)()).resolves.toBe(false),
+ );
+ await withConfig({ FIRE_PDF_BASE_URL: undefined }, () =>
+ expect(imageOcrGate(true)()).resolves.toBe(false),
);
});
});
diff --git a/apps/api/src/lib/image-ocr-gate.ts b/apps/api/src/lib/image-ocr-gate.ts
index d2c2a84f0..213352789 100644
--- a/apps/api/src/lib/image-ocr-gate.ts
+++ b/apps/api/src/lib/image-ocr-gate.ts
@@ -1,70 +1,31 @@
import { config } from "../config";
-import type { TeamFlags } from "../controllers/v1/types";
-import { getACUCTeam } from "../controllers/auth";
-import { logger } from "./logger";
/**
- * Raster image OCR rides on FirePDF and is rolled out per team through the
- * `imageOcr` team flag. Every entry point (URL-extension routing, the browser
- * handoff, parse uploads) consults this one check, so a team without the flag
- * gets exactly the pre-existing behaviour.
+ * Raster image OCR rides on FirePDF and is switched on for the whole
+ * deployment by `IMAGE_OCR_ENABLED`. Every entry point (URL-extension
+ * routing, the browser handoff, parse uploads) consults this one check, so
+ * with the switch off every request gets exactly the pre-existing
+ * unsupported-file behaviour.
*/
-export function isImageOcrEnabled(
- teamFlags: TeamFlags | null | undefined,
-): boolean {
- return !!config.FIRE_PDF_BASE_URL && teamFlags?.imageOcr === true;
+export function isImageOcrEnabled(): boolean {
+ return !!config.FIRE_PDF_BASE_URL && config.IMAGE_OCR_ENABLED;
}
-/** Per-scrape gate: resolved lazily on first call and memoized. */
+/**
+ * Per-scrape gate. Async so the callers that consult it late (the browser
+ * handoff sniff, the image engine, the index) share one shape even though
+ * the answer is known when the scrape starts.
+ */
export type ImageOcrGate = () => Promise<boolean>;
-const OFF: Promise<boolean> = Promise.resolve(false);
-
/**
* Builds the per-scrape gate: whether this request may OCR raster images.
*
* Two conditions fold into it. The request's `parsers` must include the
* `image` parser β€” it does by default, and a parse upload of an image counts
- * regardless β€” and the team must have the `imageOcr` flag while the feature
- * rolls out. A request that opted out is settled up front without any I/O.
- *
- * For the team side, single scrapes and parse uploads carry the
- * authenticated team's flags in their internalOptions and resolve without
- * I/O; batch-scrape and crawl jobs do not, so for those the flags come from
- * the cached team ACUC. That lookup is deferred until a caller actually needs
- * the answer (an image-extension URL, an image handoff, the image engine, a
- * cached image document) and memoized, so the ordinary HTML documents that
- * make up almost every crawl never pay for it. Any lookup failure keeps the
- * pre-existing behaviour.
+ * regardless β€” and the deployment switch must be on (see isImageOcrEnabled).
*/
-export function imageOcrGate(
- teamId: string | undefined,
- teamFlags: TeamFlags | null | undefined,
- requested: boolean,
-): ImageOcrGate {
- if (!requested) return () => OFF;
- let pending: Promise<boolean> | undefined;
- return () => {
- pending ??= resolveImageOcrEnabled(teamId, teamFlags);
- return pending;
- };
-}
-
-async function resolveImageOcrEnabled(
- teamId: string | undefined,
- teamFlags: TeamFlags | null | undefined,
-): Promise<boolean> {
- if (!config.FIRE_PDF_BASE_URL) return false;
- if (teamFlags !== undefined) return isImageOcrEnabled(teamFlags);
- if (!teamId) return false;
- try {
- const acuc = await getACUCTeam(teamId);
- return isImageOcrEnabled(acuc?.flags ?? null);
- } catch (error) {
- logger.warn("Failed to resolve team flags for image OCR; leaving it off", {
- teamId,
- error,
- });
- return false;
- }
+export function imageOcrGate(requested: boolean): ImageOcrGate {
+ const enabled = Promise.resolve(requested && isImageOcrEnabled());
+ return () => enabled;
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/image/index.ts b/apps/api/src/scraper/scrapeURL/engines/image/index.ts
index 7623bfb1f..9ad468510 100644
--- a/apps/api/src/scraper/scrapeURL/engines/image/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/image/index.ts
@@ -17,9 +17,10 @@ import {
* (PNG/JPEG/JPEG 2000/TIFF/GIF/BMP), finds no text layer, and runs the same
* layout + OCR path a scanned PDF page takes.
*
- * OCR is on by default and opt-out per request (a `parsers` list without
- * `image`; a parse upload of an image always counts), and rolled out per
- * team (imageOcr flag); both are folded into `meta.imageOcrEnabled`. The pdf
+ * Image OCR is off until the deployment enables it (`IMAGE_OCR_ENABLED`,
+ * default false). Once it is on, every request gets it unless it opts out
+ * per request (a `parsers` list without `image`; a parse upload of an image
+ * always counts); both conditions are folded into `meta.imageOcrEnabled`. The pdf
* parser's options (mode, maxPages, pages, blocks, pageMarkers) are not
* consulted: an image has no text layer to fall back on, so every admitted
* image is OCR'd. A caller who wants the bytes instead uses the `rawBase64`
@@ -58,10 +59,10 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {
}
// Requests already identified as images (a browser handoff, the image
- // flag, a forceEngine pin, or an image-extension URL) resolve the
- // per-team gate up front. Anything else is a waterfall tail that has to
- // sniff its bytes first, so an ordinary failed page never pays for a
- // team lookup here.
+ // flag, a forceEngine pin, or an image-extension URL) consult the gate
+ // up front. Anything else is a waterfall tail that has to sniff its
+ // bytes first, so an ordinary failed page is rejected on its bytes, not
+ // on the gate.
const knownImage =
meta.imagePrefetch != null ||
meta.featureFlags.has("image") ||
@@ -70,7 +71,7 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {
new URL(meta.rewrittenUrl ?? meta.url).pathname,
) !== null;
if (knownImage && !(await meta.imageOcrEnabled())) {
- // Opted out (parsers without image) or no team flag: the request gets
+ // Opted out (parsers without image) or switched off: the request gets
// the unsupported-file error the URL path has always produced, whose
// message names the parser.
throw new UnsupportedFileError(
@@ -132,7 +133,7 @@ export async function scrapeImage(meta: Meta): Promise<EngineScrapeResult> {
}
// A tail request whose bytes turned out to be an image: consult the gate
- // now that the lookup is warranted.
+ // now that it is warranted.
if (!knownImage && !(await meta.imageOcrEnabled())) {
throw new UnsupportedFileError(contentType);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index c9ac123f3..c44ef7f77 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -97,8 +97,8 @@ const engines: Engine[] = [
"pdf",
"document",
// Image OCR needs FirePDF; without it the engine would only be a wasted
- // tail download on every failed scrape. Per-team enablement is decided
- // where images are routed (the imageOcr team flag), not here.
+ // tail download on every failed scrape. Whether images are actually OCR'd
+ // is decided where they are routed (lib/image-ocr-gate.ts), not here.
...(config.FIRE_PDF_BASE_URL ? ["image" as const] : []),
];
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 586a55b17..08649ec09 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -118,7 +118,7 @@ async function feResToImagePrefetch(
* Sniffs a browser handoff for a raster image the image engine can OCR.
* Returns the canonical content type from the magic bytes, or null when the
* payload is missing, is not a supported format, or the request may not OCR
- * images (no image parser, or a team without the flag) β€” in which case the
+ * images (no image parser, or the switch is off) β€” in which case the
* caller falls through to the historical unsupported-file rejection.
*/
async function sniffImageHandoff(
@@ -129,8 +129,8 @@ async function sniffImageHandoff(
if (content === undefined) return null;
const contentType = sniffImageContentTypeFromBase64(content);
if (contentType === null) return null;
- // Only now consult the gate (image parser + team flag): this is the one
- // lookup an image request may cost, and non-image responses never reach it.
+ // Only now consult the gate (image parser + deployment switch); non-image
+ // responses never reach it.
return (await imageOcrEnabled()) ? contentType : null;
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 91d9b3022..c9649e6c2 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -170,10 +170,8 @@ export type Meta = {
mock: MockState | null;
/** Whether this scrape may OCR raster images: the request's parsers
* include `image` (the default; a parse upload of an image always counts)
- * and the team has the imageOcr flag with FirePDF configured. Lazy and
- * memoized: the browser handoff, the image engine and the index only ask
- * once a request actually looks like an image, so plain documents never
- * pay for the team lookup. */
+ * and the deployment has image OCR switched on with FirePDF configured.
+ * Consulted by the browser handoff, the image engine and the index. */
imageOcrEnabled: ImageOcrGate;
pdfPrefetch:
| {
@@ -346,9 +344,9 @@ function buildFeatureFlags(
flags.add("pdf");
} else if (imageExtensionFromUrlPath(lowerPath) !== null && imageOcrEnabled) {
// Raster images are OCR'd through FirePDF when the request's parsers
- // include `image` (the default) and the team has the imageOcr flag (see
- // engines/image). Everyone else stays on the ordinary waterfall and
- // fails as an unsupported file, exactly as before.
+ // include `image` (the default) and the deployment has image OCR on (see
+ // engines/image and lib/image-ocr-gate). Everything else stays on the
+ // ordinary waterfall and fails as an unsupported file, exactly as before.
flags.add("image");
}
@@ -538,16 +536,14 @@ async function buildMetaObject(
const effectiveOptions = applyScrapeOptionsDefaults(options);
// Image OCR follows the parsers option: on by default, off when the caller
// sends a list without `image`. A parse upload of an image is a request to
- // parse that file, so it counts regardless. The team flag is checked lazily
+ // parse that file, so it counts regardless. The deployment switch sits
// behind this.
const imageOcrEnabled = imageOcrGate(
- internalOptions.teamId,
- internalOptions.teamFlags,
shouldParseImages(effectiveOptions.parsers) ||
internalOptions.uploadedFile?.kind === "image",
);
// Only an image-extension URL needs the answer up front; everything else
- // resolves lazily on an image handoff, if one ever happens.
+ // consults the gate on an image handoff, if one ever happens.
const imageOcrForUrl =
imageExtensionFromUrlPath(new URL(url).pathname) !== null &&
(await imageOcrEnabled());