Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add GET /v2/parse/formats (#4905)

Gergő Móricza9885f490342
diff --git a/apps/api/src/__tests__/snips/v2/parse-formats.test.ts b/apps/api/src/__tests__/snips/v2/parse-formats.test.ts
new file mode 100644
index 000000000..98ee3e4a4
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/parse-formats.test.ts
@@ -0,0 +1,150 @@
+import { describeIf, TEST_PRODUCTION } from "../lib";
+import request, { idmux, Identity, scrapeTimeout, TEST_API_URL } from "./lib";
+
+type ParseFormat = {
+ format: string;
+ kind: "document" | "image";
+ extensions: string[];
+ mimeTypes: string[];
+ available: boolean;
+};
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "parse-formats",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+async function getParseFormats(): Promise<ParseFormat[]> {
+ const response = await request(TEST_API_URL)
+ .get("/v2/parse/formats")
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+
+ expect(response.statusCode, JSON.stringify(response.body)).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.headers["cache-control"]).toBe("private, max-age=3600");
+ return response.body.data.formats;
+}
+
+// screenshot output is rejected after the upload type is accepted, so a
+// PARSE_UNSUPPORTED_OPTIONS answer proves the type passed validation without
+// running a parse.
+async function parseValidationCode(filename: string, contentType: string) {
+ const response = await request(TEST_API_URL)
+ .post("/v2/parse")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .attach("file", Buffer.from("format-probe"), { filename, contentType })
+ .field("options", JSON.stringify({ formats: ["screenshot"] }));
+
+ expect(response.statusCode, JSON.stringify(response.body)).toBe(400);
+ return response.body.code as string;
+}
+
+describe("GET /v2/parse/formats", () => {
+ it(
+ "lists known document and image formats",
+ async () => {
+ const formats = await getParseFormats();
+ const byName = new Map(formats.map(format => [format.format, format]));
+
+ expect(byName.get("pdf")).toMatchObject({
+ kind: "document",
+ available: true,
+ extensions: [".pdf"],
+ mimeTypes: ["application/pdf"],
+ });
+ expect(byName.get("docx")).toMatchObject({
+ kind: "document",
+ available: true,
+ extensions: [".docx"],
+ mimeTypes: [
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ ],
+ });
+ expect(byName.get("png")).toMatchObject({
+ kind: "image",
+ extensions: [".png"],
+ mimeTypes: ["image/png"],
+ });
+ expect(byName.get("jpg")?.extensions).toEqual(
+ expect.arrayContaining([".jpg", ".jpeg"]),
+ );
+
+ for (const format of formats) {
+ expect(format.extensions.length).toBeGreaterThan(0);
+ expect(format.mimeTypes.length).toBeGreaterThan(0);
+ expect(typeof format.available).toBe("boolean");
+ if (format.kind === "document") expect(format.available).toBe(true);
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "matches what /v2/parse accepts for every listed extension and MIME type",
+ async () => {
+ const formats = await getParseFormats();
+ const probes = formats.flatMap(format => [
+ ...format.extensions.map(extension => ({
+ format,
+ filename: `probe${extension}`,
+ contentType: "application/octet-stream",
+ })),
+ ...format.mimeTypes.map(mimeType => ({
+ format,
+ filename: "probe",
+ contentType: mimeType,
+ })),
+ ]);
+
+ for (const probe of probes) {
+ const code = await parseValidationCode(
+ probe.filename,
+ probe.contentType,
+ );
+ expect({ ...probe, code }).toEqual({
+ ...probe,
+ code: probe.format.available
+ ? "PARSE_UNSUPPORTED_OPTIONS"
+ : "UNSUPPORTED_FILE_TYPE",
+ });
+ }
+ },
+ scrapeTimeout * 2,
+ );
+
+ it(
+ "rejects the wrong method",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/parse/formats")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .send({});
+
+ expect(response.statusCode).toBe(405);
+ expect(response.body.success).toBe(false);
+ expect(response.body.code).toBe("METHOD_NOT_ALLOWED");
+ expect(response.body.allowed_methods).toContain("GET");
+ },
+ scrapeTimeout,
+ );
+
+ describeIf(TEST_PRODUCTION)("auth", () => {
+ it(
+ "rejects an invalid API key",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .get("/v2/parse/formats")
+ .set("Authorization", "Bearer fc-invalid-api-key");
+
+ expect(response.statusCode).toBe(401);
+ expect(response.body.success).toBe(false);
+ },
+ scrapeTimeout,
+ );
+ });
+});
diff --git a/apps/api/src/controllers/v2/parse-formats.ts b/apps/api/src/controllers/v2/parse-formats.ts
new file mode 100644
index 000000000..84a41bc29
--- /dev/null
+++ b/apps/api/src/controllers/v2/parse-formats.ts
@@ -0,0 +1,20 @@
+import { Response } from "express";
+import { RequestWithAuth } from "./types";
+import { isImageOcrEnabled } from "../../lib/image-ocr-gate";
+import { listParseFormats, ParseFormat } from "../../lib/parse-formats";
+
+type ParseFormatsResponse = {
+ success: true;
+ data: { formats: ParseFormat[] };
+};
+
+export async function parseFormatsController(
+ _req: RequestWithAuth<{}, ParseFormatsResponse, undefined>,
+ res: Response<ParseFormatsResponse>,
+) {
+ res.setHeader("Cache-Control", "private, max-age=3600");
+ return res.status(200).json({
+ success: true,
+ data: { formats: listParseFormats(isImageOcrEnabled()) },
+ });
+}
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 9d0b563f7..042646e04 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -46,17 +46,24 @@ import {
} from "../../lib/image-formats";
import { isImageOcrEnabled } from "../../lib/image-ocr-gate";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
+import {
+ HTML_CONTENT_TYPES,
+ HTML_EXTENSIONS,
+ listParseFormats,
+ PDF_CONTENT_TYPES,
+ PDF_EXTENSIONS,
+} from "../../lib/parse-formats";
const AGENT_INTEROP_CONCURRENCY_BOOST = 3;
-const BASE_PARSE_FILE_TYPES =
- ".html, .htm, .xhtml, .pdf, .docx, .doc, .docm, .odt, .ods, .odp, .rtf, .xlsx, .xls, .xlsm, .xlsb, .pptx, .ppt, .pptm, .epub, .csv";
/** Image uploads are OCR'd through FirePDF, so they are only advertised as
* supported for teams with image OCR enabled (matching
* detectUploadedFileKind). */
export function getSupportedParseFileTypes(imageOcrEnabled: boolean): string {
- if (!imageOcrEnabled) return BASE_PARSE_FILE_TYPES;
- return `${BASE_PARSE_FILE_TYPES}, ${[...IMAGE_EXTENSIONS].sort().join(", ")}`;
+ return listParseFormats(imageOcrEnabled)
+ .filter(format => format.available)
+ .flatMap(format => format.extensions)
+ .join(", ");
}
export function detectUploadedFileKind(
@@ -68,9 +75,10 @@ export function detectUploadedFileKind(
const normalizedType = contentType?.toLowerCase() ?? "";
const isPdf =
- extension === ".pdf" ||
- normalizedType === "application/pdf" ||
- normalizedType.startsWith("application/pdf;");
+ PDF_EXTENSIONS.includes(extension) ||
+ PDF_CONTENT_TYPES.some(
+ type => normalizedType === type || normalizedType.startsWith(`${type};`),
+ );
if (isPdf) {
return "pdf";
@@ -96,11 +104,8 @@ export function detectUploadedFileKind(
}
const isHtml =
- extension === ".html" ||
- extension === ".htm" ||
- extension === ".xhtml" ||
- normalizedType.includes("text/html") ||
- normalizedType.includes("application/xhtml+xml");
+ HTML_EXTENSIONS.includes(extension) ||
+ HTML_CONTENT_TYPES.some(type => normalizedType.includes(type));
if (isHtml) {
return "html";
diff --git a/apps/api/src/lib/document-formats.ts b/apps/api/src/lib/document-formats.ts
index bce80f84a..749ebaefa 100644
--- a/apps/api/src/lib/document-formats.ts
+++ b/apps/api/src/lib/document-formats.ts
@@ -1,36 +1,39 @@
// Document formats handled by the document engine (anydoc). PDF is
// intentionally absent — it has its own engine with OCR support.
-const CONTENT_TYPE_TO_EXTENSION = new Map<string, string>([
- [
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- ".docx",
- ],
- ["application/vnd.ms-word.document.macroenabled.12", ".docm"],
- ["application/msword", ".doc"],
- [
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- ".xlsx",
- ],
- ["application/vnd.ms-excel", ".xls"],
- ["application/vnd.ms-excel.sheet.macroenabled.12", ".xlsm"],
- ["application/vnd.ms-excel.sheet.binary.macroenabled.12", ".xlsb"],
- [
- "application/vnd.openxmlformats-officedocument.presentationml.presentation",
- ".pptx",
- ],
- ["application/vnd.ms-powerpoint", ".ppt"],
- ["application/vnd.ms-powerpoint.presentation.macroenabled.12", ".pptm"],
- ["application/vnd.oasis.opendocument.text", ".odt"],
- ["application/vnd.oasis.opendocument.spreadsheet", ".ods"],
- ["application/vnd.oasis.opendocument.presentation", ".odp"],
- ["application/rtf", ".rtf"],
- ["text/rtf", ".rtf"],
- ["application/epub+zip", ".epub"],
- ["text/csv", ".csv"],
-]);
+export const DOCUMENT_CONTENT_TYPE_TO_EXTENSION: ReadonlyMap<string, string> =
+ new Map<string, string>([
+ [
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ ".docx",
+ ],
+ ["application/vnd.ms-word.document.macroenabled.12", ".docm"],
+ ["application/msword", ".doc"],
+ [
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ ".xlsx",
+ ],
+ ["application/vnd.ms-excel", ".xls"],
+ ["application/vnd.ms-excel.sheet.macroenabled.12", ".xlsm"],
+ ["application/vnd.ms-excel.sheet.binary.macroenabled.12", ".xlsb"],
+ [
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+ ".pptx",
+ ],
+ ["application/vnd.ms-powerpoint", ".ppt"],
+ ["application/vnd.ms-powerpoint.presentation.macroenabled.12", ".pptm"],
+ ["application/vnd.oasis.opendocument.text", ".odt"],
+ ["application/vnd.oasis.opendocument.spreadsheet", ".ods"],
+ ["application/vnd.oasis.opendocument.presentation", ".odp"],
+ ["application/rtf", ".rtf"],
+ ["text/rtf", ".rtf"],
+ ["application/epub+zip", ".epub"],
+ ["text/csv", ".csv"],
+ ]);
-export const DOCUMENT_EXTENSIONS = new Set(CONTENT_TYPE_TO_EXTENSION.values());
+export const DOCUMENT_EXTENSIONS = new Set(
+ DOCUMENT_CONTENT_TYPE_TO_EXTENSION.values(),
+);
// Longest first so ".docx" wins over ".doc" when matching URL paths.
const EXTENSIONS_BY_LENGTH = [...DOCUMENT_EXTENSIONS].sort(
@@ -42,14 +45,14 @@ export function documentExtensionFromContentType(
): string | null {
if (!contentType) return null;
const mediaType = contentType.split(";")[0].trim().toLowerCase();
- return CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;
+ return DOCUMENT_CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;
}
export function documentContentTypeFromExtension(
extension: string,
): string | null {
const ext = extension.toLowerCase();
- for (const [contentType, mapped] of CONTENT_TYPE_TO_EXTENSION) {
+ for (const [contentType, mapped] of DOCUMENT_CONTENT_TYPE_TO_EXTENSION) {
if (mapped === ext) return contentType;
}
return null;
diff --git a/apps/api/src/lib/image-formats.ts b/apps/api/src/lib/image-formats.ts
index 42694f8ce..4a33e6477 100644
--- a/apps/api/src/lib/image-formats.ts
+++ b/apps/api/src/lib/image-formats.ts
@@ -5,34 +5,38 @@
// them, so they keep failing fast as unsupported files instead of burning a
// round trip.
-const CONTENT_TYPE_TO_EXTENSION = new Map<string, string>([
- ["image/png", ".png"],
- ["image/jpeg", ".jpg"],
- // Non-standard but common in the wild.
- ["image/jpg", ".jpg"],
- ["image/pjpeg", ".jpg"],
- ["image/tiff", ".tif"],
- ["image/x-tiff", ".tif"],
- ["image/gif", ".gif"],
- ["image/bmp", ".bmp"],
- ["image/x-ms-bmp", ".bmp"],
- // JPEG 2000: the JP2 container digitized book scans and IIIF tile sources
- // are served in, and its extended JPX sibling. Raw codestreams (.j2k,
- // .j2c) share the type: FirePDF opens them the same way.
- ["image/jp2", ".jp2"],
- ["image/jpx", ".jp2"],
- // Raw codestreams have no registered type; these are the labels servers
- // use for them.
- ["image/j2k", ".jp2"],
- ["image/j2c", ".jp2"],
- ["image/x-j2c", ".jp2"],
- // WebP and AVIF: what CDNs serve to a browser that advertises them in
- // Accept, whatever the URL's extension says. FirePDF transcodes both.
- ["image/webp", ".webp"],
- ["image/avif", ".avif"],
-]);
+export const IMAGE_CONTENT_TYPE_TO_EXTENSION: ReadonlyMap<string, string> =
+ new Map<string, string>([
+ ["image/png", ".png"],
+ ["image/jpeg", ".jpg"],
+ // Non-standard but common in the wild.
+ ["image/jpg", ".jpg"],
+ ["image/pjpeg", ".jpg"],
+ ["image/tiff", ".tif"],
+ ["image/x-tiff", ".tif"],
+ ["image/gif", ".gif"],
+ ["image/bmp", ".bmp"],
+ ["image/x-ms-bmp", ".bmp"],
+ // JPEG 2000: the JP2 container digitized book scans and IIIF tile sources
+ // are served in, and its extended JPX sibling. Raw codestreams (.j2k,
+ // .j2c) share the type: FirePDF opens them the same way.
+ ["image/jp2", ".jp2"],
+ ["image/jpx", ".jp2"],
+ // Raw codestreams have no registered type; these are the labels servers
+ // use for them.
+ ["image/j2k", ".jp2"],
+ ["image/j2c", ".jp2"],
+ ["image/x-j2c", ".jp2"],
+ // WebP and AVIF: what CDNs serve to a browser that advertises them in
+ // Accept, whatever the URL's extension says. FirePDF transcodes both.
+ ["image/webp", ".webp"],
+ ["image/avif", ".avif"],
+ ]);
-const EXTENSION_ALIASES = new Map<string, string>([
+export const IMAGE_EXTENSION_ALIASES: ReadonlyMap<string, string> = new Map<
+ string,
+ string
+>([
[".jpeg", ".jpg"],
[".tiff", ".tif"],
[".jpx", ".jp2"],
@@ -42,8 +46,8 @@ const EXTENSION_ALIASES = new Map<string, string>([
]);
export const IMAGE_EXTENSIONS = new Set([
- ...CONTENT_TYPE_TO_EXTENSION.values(),
- ...EXTENSION_ALIASES.keys(),
+ ...IMAGE_CONTENT_TYPE_TO_EXTENSION.values(),
+ ...IMAGE_EXTENSION_ALIASES.keys(),
]);
export function imageExtensionFromContentType(
@@ -51,15 +55,15 @@ export function imageExtensionFromContentType(
): string | null {
if (!contentType) return null;
const mediaType = contentType.split(";")[0].trim().toLowerCase();
- return CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;
+ return IMAGE_CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;
}
export function imageContentTypeFromExtension(
extension: string,
): string | null {
const lower = extension.toLowerCase();
- const ext = EXTENSION_ALIASES.get(lower) ?? lower;
- for (const [contentType, mapped] of CONTENT_TYPE_TO_EXTENSION) {
+ const ext = IMAGE_EXTENSION_ALIASES.get(lower) ?? lower;
+ for (const [contentType, mapped] of IMAGE_CONTENT_TYPE_TO_EXTENSION) {
if (mapped === ext) return contentType;
}
return null;
diff --git a/apps/api/src/lib/parse-formats.ts b/apps/api/src/lib/parse-formats.ts
new file mode 100644
index 000000000..4378c313f
--- /dev/null
+++ b/apps/api/src/lib/parse-formats.ts
@@ -0,0 +1,73 @@
+import { DOCUMENT_CONTENT_TYPE_TO_EXTENSION } from "./document-formats";
+import {
+ IMAGE_CONTENT_TYPE_TO_EXTENSION,
+ IMAGE_EXTENSION_ALIASES,
+} from "./image-formats";
+
+export const PDF_EXTENSIONS = [".pdf"];
+export const PDF_CONTENT_TYPES = ["application/pdf"];
+export const HTML_EXTENSIONS = [".html", ".htm", ".xhtml"];
+export const HTML_CONTENT_TYPES = ["text/html", "application/xhtml+xml"];
+
+export type ParseFormat = {
+ format: string;
+ kind: "document" | "image";
+ extensions: string[];
+ mimeTypes: string[];
+ available: boolean;
+};
+
+function groupByExtension(
+ kind: ParseFormat["kind"],
+ available: boolean,
+ contentTypeToExtension: ReadonlyMap<string, string>,
+ aliases: ReadonlyMap<string, string> = new Map(),
+): ParseFormat[] {
+ const formats = new Map<string, ParseFormat>();
+ for (const [mimeType, extension] of contentTypeToExtension) {
+ const format = formats.get(extension) ?? {
+ format: extension.slice(1),
+ kind,
+ extensions: [extension],
+ mimeTypes: [],
+ available,
+ };
+ format.mimeTypes.push(mimeType);
+ formats.set(extension, format);
+ }
+ for (const [alias, extension] of aliases) {
+ formats.get(extension)?.extensions.push(alias);
+ }
+ return [...formats.values()];
+}
+
+/**
+ * Every upload type /v2/parse accepts, built from the same constants
+ * detectUploadedFileKind validates against. Image formats are only
+ * `available` where image OCR is enabled.
+ */
+export function listParseFormats(imageOcrEnabled: boolean): ParseFormat[] {
+ return [
+ {
+ format: "html",
+ kind: "document",
+ extensions: [...HTML_EXTENSIONS],
+ mimeTypes: [...HTML_CONTENT_TYPES],
+ available: true,
+ },
+ {
+ format: "pdf",
+ kind: "document",
+ extensions: [...PDF_EXTENSIONS],
+ mimeTypes: [...PDF_CONTENT_TYPES],
+ available: true,
+ },
+ ...groupByExtension("document", true, DOCUMENT_CONTENT_TYPE_TO_EXTENSION),
+ ...groupByExtension(
+ "image",
+ imageOcrEnabled,
+ IMAGE_CONTENT_TYPE_TO_EXTENSION,
+ IMAGE_EXTENSION_ALIASES,
+ ),
+ ];
+}
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 1fe51a759..936c6fef0 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -24,6 +24,7 @@ import {
parseUploadRefPayloadMiddleware,
parseUploadUrlController,
} from "../controllers/v2/parse-upload";
+import { parseFormatsController } from "../controllers/v2/parse-formats";
import { batchScrapeController } from "../controllers/v2/batch-scrape";
import { crawlController } from "../controllers/v2/crawl";
import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preview";
@@ -213,6 +214,12 @@ v2Router.post(
wrap(feedbackController),
);
+v2Router.get(
+ "/parse/formats",
+ authMiddleware(RateLimiterMode.Account),
+ wrap(parseFormatsController),
+);
+
v2Router.post(
"/parse/upload-url",
authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),