Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add GET /v2/parse/formats (#4905)
Gergő Móricza9885f490342
diff --git a/apps/api/src/__tests__/snips/v2/parse-formats.test.ts b/apps/api/src/__tests__/snips/v2/parse-formats.test.tsnew file mode 100644index 000000000..98ee3e4a4--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/parse-formats.test.ts@@ -0,0 +1,150 @@+import { describeIf, TEST_PRODUCTION } from "../lib";+import request, { idmux, Identity, scrapeTimeout, TEST_API_URL } from "./lib";++type ParseFormat = {+ format: string;+ kind: "document" | "image";+ extensions: string[];+ mimeTypes: string[];+ available: boolean;+};++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "parse-formats",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++async function getParseFormats(): Promise<ParseFormat[]> {+ const response = await request(TEST_API_URL)+ .get("/v2/parse/formats")+ .set("Authorization", `Bearer ${identity.apiKey}`);++ expect(response.statusCode, JSON.stringify(response.body)).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.headers["cache-control"]).toBe("private, max-age=3600");+ return response.body.data.formats;+}++// screenshot output is rejected after the upload type is accepted, so a+// PARSE_UNSUPPORTED_OPTIONS answer proves the type passed validation without+// running a parse.+async function parseValidationCode(filename: string, contentType: string) {+ const response = await request(TEST_API_URL)+ .post("/v2/parse")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .attach("file", Buffer.from("format-probe"), { filename, contentType })+ .field("options", JSON.stringify({ formats: ["screenshot"] }));++ expect(response.statusCode, JSON.stringify(response.body)).toBe(400);+ return response.body.code as string;+}++describe("GET /v2/parse/formats", () => {+ it(+ "lists known document and image formats",+ async () => {+ const formats = await getParseFormats();+ const byName = new Map(formats.map(format => [format.format, format]));++ expect(byName.get("pdf")).toMatchObject({+ kind: "document",+ available: true,+ extensions: [".pdf"],+ mimeTypes: ["application/pdf"],+ });+ expect(byName.get("docx")).toMatchObject({+ kind: "document",+ available: true,+ extensions: [".docx"],+ mimeTypes: [+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ ],+ });+ expect(byName.get("png")).toMatchObject({+ kind: "image",+ extensions: [".png"],+ mimeTypes: ["image/png"],+ });+ expect(byName.get("jpg")?.extensions).toEqual(+ expect.arrayContaining([".jpg", ".jpeg"]),+ );++ for (const format of formats) {+ expect(format.extensions.length).toBeGreaterThan(0);+ expect(format.mimeTypes.length).toBeGreaterThan(0);+ expect(typeof format.available).toBe("boolean");+ if (format.kind === "document") expect(format.available).toBe(true);+ }+ },+ scrapeTimeout,+ );++ it(+ "matches what /v2/parse accepts for every listed extension and MIME type",+ async () => {+ const formats = await getParseFormats();+ const probes = formats.flatMap(format => [+ ...format.extensions.map(extension => ({+ format,+ filename: `probe${extension}`,+ contentType: "application/octet-stream",+ })),+ ...format.mimeTypes.map(mimeType => ({+ format,+ filename: "probe",+ contentType: mimeType,+ })),+ ]);++ for (const probe of probes) {+ const code = await parseValidationCode(+ probe.filename,+ probe.contentType,+ );+ expect({ ...probe, code }).toEqual({+ ...probe,+ code: probe.format.available+ ? "PARSE_UNSUPPORTED_OPTIONS"+ : "UNSUPPORTED_FILE_TYPE",+ });+ }+ },+ scrapeTimeout * 2,+ );++ it(+ "rejects the wrong method",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/parse/formats")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .send({});++ expect(response.statusCode).toBe(405);+ expect(response.body.success).toBe(false);+ expect(response.body.code).toBe("METHOD_NOT_ALLOWED");+ expect(response.body.allowed_methods).toContain("GET");+ },+ scrapeTimeout,+ );++ describeIf(TEST_PRODUCTION)("auth", () => {+ it(+ "rejects an invalid API key",+ async () => {+ const response = await request(TEST_API_URL)+ .get("/v2/parse/formats")+ .set("Authorization", "Bearer fc-invalid-api-key");++ expect(response.statusCode).toBe(401);+ expect(response.body.success).toBe(false);+ },+ scrapeTimeout,+ );+ });+});diff --git a/apps/api/src/controllers/v2/parse-formats.ts b/apps/api/src/controllers/v2/parse-formats.tsnew file mode 100644index 000000000..84a41bc29--- /dev/null+++ b/apps/api/src/controllers/v2/parse-formats.ts@@ -0,0 +1,20 @@+import { Response } from "express";+import { RequestWithAuth } from "./types";+import { isImageOcrEnabled } from "../../lib/image-ocr-gate";+import { listParseFormats, ParseFormat } from "../../lib/parse-formats";++type ParseFormatsResponse = {+ success: true;+ data: { formats: ParseFormat[] };+};++export async function parseFormatsController(+ _req: RequestWithAuth<{}, ParseFormatsResponse, undefined>,+ res: Response<ParseFormatsResponse>,+) {+ res.setHeader("Cache-Control", "private, max-age=3600");+ return res.status(200).json({+ success: true,+ data: { formats: listParseFormats(isImageOcrEnabled()) },+ });+}diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 9d0b563f7..042646e04 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -46,17 +46,24 @@ import {} from "../../lib/image-formats";import { isImageOcrEnabled } from "../../lib/image-ocr-gate";import { isAgentInteropSecretValid } from "../../lib/agent-interop";+import {+ HTML_CONTENT_TYPES,+ HTML_EXTENSIONS,+ listParseFormats,+ PDF_CONTENT_TYPES,+ PDF_EXTENSIONS,+} from "../../lib/parse-formats";const AGENT_INTEROP_CONCURRENCY_BOOST = 3;-const BASE_PARSE_FILE_TYPES =- ".html, .htm, .xhtml, .pdf, .docx, .doc, .docm, .odt, .ods, .odp, .rtf, .xlsx, .xls, .xlsm, .xlsb, .pptx, .ppt, .pptm, .epub, .csv";/** Image uploads are OCR'd through FirePDF, so they are only advertised as* supported for teams with image OCR enabled (matching* detectUploadedFileKind). */export function getSupportedParseFileTypes(imageOcrEnabled: boolean): string {- if (!imageOcrEnabled) return BASE_PARSE_FILE_TYPES;- return `${BASE_PARSE_FILE_TYPES}, ${[...IMAGE_EXTENSIONS].sort().join(", ")}`;+ return listParseFormats(imageOcrEnabled)+ .filter(format => format.available)+ .flatMap(format => format.extensions)+ .join(", ");}export function detectUploadedFileKind(@@ -68,9 +75,10 @@ export function detectUploadedFileKind(const normalizedType = contentType?.toLowerCase() ?? "";const isPdf =- extension === ".pdf" ||- normalizedType === "application/pdf" ||- normalizedType.startsWith("application/pdf;");+ PDF_EXTENSIONS.includes(extension) ||+ PDF_CONTENT_TYPES.some(+ type => normalizedType === type || normalizedType.startsWith(`${type};`),+ );if (isPdf) {return "pdf";@@ -96,11 +104,8 @@ export function detectUploadedFileKind(}const isHtml =- extension === ".html" ||- extension === ".htm" ||- extension === ".xhtml" ||- normalizedType.includes("text/html") ||- normalizedType.includes("application/xhtml+xml");+ HTML_EXTENSIONS.includes(extension) ||+ HTML_CONTENT_TYPES.some(type => normalizedType.includes(type));if (isHtml) {return "html";diff --git a/apps/api/src/lib/document-formats.ts b/apps/api/src/lib/document-formats.tsindex bce80f84a..749ebaefa 100644--- a/apps/api/src/lib/document-formats.ts+++ b/apps/api/src/lib/document-formats.ts@@ -1,36 +1,39 @@// Document formats handled by the document engine (anydoc). PDF is// intentionally absent — it has its own engine with OCR support.-const CONTENT_TYPE_TO_EXTENSION = new Map<string, string>([- [- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",- ".docx",- ],- ["application/vnd.ms-word.document.macroenabled.12", ".docm"],- ["application/msword", ".doc"],- [- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",- ".xlsx",- ],- ["application/vnd.ms-excel", ".xls"],- ["application/vnd.ms-excel.sheet.macroenabled.12", ".xlsm"],- ["application/vnd.ms-excel.sheet.binary.macroenabled.12", ".xlsb"],- [- "application/vnd.openxmlformats-officedocument.presentationml.presentation",- ".pptx",- ],- ["application/vnd.ms-powerpoint", ".ppt"],- ["application/vnd.ms-powerpoint.presentation.macroenabled.12", ".pptm"],- ["application/vnd.oasis.opendocument.text", ".odt"],- ["application/vnd.oasis.opendocument.spreadsheet", ".ods"],- ["application/vnd.oasis.opendocument.presentation", ".odp"],- ["application/rtf", ".rtf"],- ["text/rtf", ".rtf"],- ["application/epub+zip", ".epub"],- ["text/csv", ".csv"],-]);+export const DOCUMENT_CONTENT_TYPE_TO_EXTENSION: ReadonlyMap<string, string> =+ new Map<string, string>([+ [+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ ".docx",+ ],+ ["application/vnd.ms-word.document.macroenabled.12", ".docm"],+ ["application/msword", ".doc"],+ [+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ ".xlsx",+ ],+ ["application/vnd.ms-excel", ".xls"],+ ["application/vnd.ms-excel.sheet.macroenabled.12", ".xlsm"],+ ["application/vnd.ms-excel.sheet.binary.macroenabled.12", ".xlsb"],+ [+ "application/vnd.openxmlformats-officedocument.presentationml.presentation",+ ".pptx",+ ],+ ["application/vnd.ms-powerpoint", ".ppt"],+ ["application/vnd.ms-powerpoint.presentation.macroenabled.12", ".pptm"],+ ["application/vnd.oasis.opendocument.text", ".odt"],+ ["application/vnd.oasis.opendocument.spreadsheet", ".ods"],+ ["application/vnd.oasis.opendocument.presentation", ".odp"],+ ["application/rtf", ".rtf"],+ ["text/rtf", ".rtf"],+ ["application/epub+zip", ".epub"],+ ["text/csv", ".csv"],+ ]);-export const DOCUMENT_EXTENSIONS = new Set(CONTENT_TYPE_TO_EXTENSION.values());+export const DOCUMENT_EXTENSIONS = new Set(+ DOCUMENT_CONTENT_TYPE_TO_EXTENSION.values(),+);// Longest first so ".docx" wins over ".doc" when matching URL paths.const EXTENSIONS_BY_LENGTH = [...DOCUMENT_EXTENSIONS].sort(@@ -42,14 +45,14 @@ export function documentExtensionFromContentType(): string | null {if (!contentType) return null;const mediaType = contentType.split(";")[0].trim().toLowerCase();- return CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;+ return DOCUMENT_CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;}export function documentContentTypeFromExtension(extension: string,): string | null {const ext = extension.toLowerCase();- for (const [contentType, mapped] of CONTENT_TYPE_TO_EXTENSION) {+ for (const [contentType, mapped] of DOCUMENT_CONTENT_TYPE_TO_EXTENSION) {if (mapped === ext) return contentType;}return null;diff --git a/apps/api/src/lib/image-formats.ts b/apps/api/src/lib/image-formats.tsindex 42694f8ce..4a33e6477 100644--- a/apps/api/src/lib/image-formats.ts+++ b/apps/api/src/lib/image-formats.ts@@ -5,34 +5,38 @@// them, so they keep failing fast as unsupported files instead of burning a// round trip.-const CONTENT_TYPE_TO_EXTENSION = new Map<string, string>([- ["image/png", ".png"],- ["image/jpeg", ".jpg"],- // Non-standard but common in the wild.- ["image/jpg", ".jpg"],- ["image/pjpeg", ".jpg"],- ["image/tiff", ".tif"],- ["image/x-tiff", ".tif"],- ["image/gif", ".gif"],- ["image/bmp", ".bmp"],- ["image/x-ms-bmp", ".bmp"],- // JPEG 2000: the JP2 container digitized book scans and IIIF tile sources- // are served in, and its extended JPX sibling. Raw codestreams (.j2k,- // .j2c) share the type: FirePDF opens them the same way.- ["image/jp2", ".jp2"],- ["image/jpx", ".jp2"],- // Raw codestreams have no registered type; these are the labels servers- // use for them.- ["image/j2k", ".jp2"],- ["image/j2c", ".jp2"],- ["image/x-j2c", ".jp2"],- // WebP and AVIF: what CDNs serve to a browser that advertises them in- // Accept, whatever the URL's extension says. FirePDF transcodes both.- ["image/webp", ".webp"],- ["image/avif", ".avif"],-]);+export const IMAGE_CONTENT_TYPE_TO_EXTENSION: ReadonlyMap<string, string> =+ new Map<string, string>([+ ["image/png", ".png"],+ ["image/jpeg", ".jpg"],+ // Non-standard but common in the wild.+ ["image/jpg", ".jpg"],+ ["image/pjpeg", ".jpg"],+ ["image/tiff", ".tif"],+ ["image/x-tiff", ".tif"],+ ["image/gif", ".gif"],+ ["image/bmp", ".bmp"],+ ["image/x-ms-bmp", ".bmp"],+ // JPEG 2000: the JP2 container digitized book scans and IIIF tile sources+ // are served in, and its extended JPX sibling. Raw codestreams (.j2k,+ // .j2c) share the type: FirePDF opens them the same way.+ ["image/jp2", ".jp2"],+ ["image/jpx", ".jp2"],+ // Raw codestreams have no registered type; these are the labels servers+ // use for them.+ ["image/j2k", ".jp2"],+ ["image/j2c", ".jp2"],+ ["image/x-j2c", ".jp2"],+ // WebP and AVIF: what CDNs serve to a browser that advertises them in+ // Accept, whatever the URL's extension says. FirePDF transcodes both.+ ["image/webp", ".webp"],+ ["image/avif", ".avif"],+ ]);-const EXTENSION_ALIASES = new Map<string, string>([+export const IMAGE_EXTENSION_ALIASES: ReadonlyMap<string, string> = new Map<+ string,+ string+>([[".jpeg", ".jpg"],[".tiff", ".tif"],[".jpx", ".jp2"],@@ -42,8 +46,8 @@ const EXTENSION_ALIASES = new Map<string, string>([]);export const IMAGE_EXTENSIONS = new Set([- ...CONTENT_TYPE_TO_EXTENSION.values(),- ...EXTENSION_ALIASES.keys(),+ ...IMAGE_CONTENT_TYPE_TO_EXTENSION.values(),+ ...IMAGE_EXTENSION_ALIASES.keys(),]);export function imageExtensionFromContentType(@@ -51,15 +55,15 @@ export function imageExtensionFromContentType(): string | null {if (!contentType) return null;const mediaType = contentType.split(";")[0].trim().toLowerCase();- return CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;+ return IMAGE_CONTENT_TYPE_TO_EXTENSION.get(mediaType) ?? null;}export function imageContentTypeFromExtension(extension: string,): string | null {const lower = extension.toLowerCase();- const ext = EXTENSION_ALIASES.get(lower) ?? lower;- for (const [contentType, mapped] of CONTENT_TYPE_TO_EXTENSION) {+ const ext = IMAGE_EXTENSION_ALIASES.get(lower) ?? lower;+ for (const [contentType, mapped] of IMAGE_CONTENT_TYPE_TO_EXTENSION) {if (mapped === ext) return contentType;}return null;diff --git a/apps/api/src/lib/parse-formats.ts b/apps/api/src/lib/parse-formats.tsnew file mode 100644index 000000000..4378c313f--- /dev/null+++ b/apps/api/src/lib/parse-formats.ts@@ -0,0 +1,73 @@+import { DOCUMENT_CONTENT_TYPE_TO_EXTENSION } from "./document-formats";+import {+ IMAGE_CONTENT_TYPE_TO_EXTENSION,+ IMAGE_EXTENSION_ALIASES,+} from "./image-formats";++export const PDF_EXTENSIONS = [".pdf"];+export const PDF_CONTENT_TYPES = ["application/pdf"];+export const HTML_EXTENSIONS = [".html", ".htm", ".xhtml"];+export const HTML_CONTENT_TYPES = ["text/html", "application/xhtml+xml"];++export type ParseFormat = {+ format: string;+ kind: "document" | "image";+ extensions: string[];+ mimeTypes: string[];+ available: boolean;+};++function groupByExtension(+ kind: ParseFormat["kind"],+ available: boolean,+ contentTypeToExtension: ReadonlyMap<string, string>,+ aliases: ReadonlyMap<string, string> = new Map(),+): ParseFormat[] {+ const formats = new Map<string, ParseFormat>();+ for (const [mimeType, extension] of contentTypeToExtension) {+ const format = formats.get(extension) ?? {+ format: extension.slice(1),+ kind,+ extensions: [extension],+ mimeTypes: [],+ available,+ };+ format.mimeTypes.push(mimeType);+ formats.set(extension, format);+ }+ for (const [alias, extension] of aliases) {+ formats.get(extension)?.extensions.push(alias);+ }+ return [...formats.values()];+}++/**+ * Every upload type /v2/parse accepts, built from the same constants+ * detectUploadedFileKind validates against. Image formats are only+ * `available` where image OCR is enabled.+ */+export function listParseFormats(imageOcrEnabled: boolean): ParseFormat[] {+ return [+ {+ format: "html",+ kind: "document",+ extensions: [...HTML_EXTENSIONS],+ mimeTypes: [...HTML_CONTENT_TYPES],+ available: true,+ },+ {+ format: "pdf",+ kind: "document",+ extensions: [...PDF_EXTENSIONS],+ mimeTypes: [...PDF_CONTENT_TYPES],+ available: true,+ },+ ...groupByExtension("document", true, DOCUMENT_CONTENT_TYPE_TO_EXTENSION),+ ...groupByExtension(+ "image",+ imageOcrEnabled,+ IMAGE_CONTENT_TYPE_TO_EXTENSION,+ IMAGE_EXTENSION_ALIASES,+ ),+ ];+}diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 1fe51a759..936c6fef0 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -24,6 +24,7 @@ import {parseUploadRefPayloadMiddleware,parseUploadUrlController,} from "../controllers/v2/parse-upload";+import { parseFormatsController } from "../controllers/v2/parse-formats";import { batchScrapeController } from "../controllers/v2/batch-scrape";import { crawlController } from "../controllers/v2/crawl";import { crawlParamsPreviewController } from "../controllers/v2/crawl-params-preview";@@ -213,6 +214,12 @@ v2Router.post(wrap(feedbackController),);+v2Router.get(+ "/parse/formats",+ authMiddleware(RateLimiterMode.Account),+ wrap(parseFormatsController),+);+v2Router.post("/parse/upload-url",authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),