Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

ENG-3088: Change parsers parameter from object to array format (#1931)

Gergő Móriczcb1415aa203f
diff --git a/apps/api/src/__tests__/snips/v2/billing.test.ts b/apps/api/src/__tests__/snips/v2/billing.test.ts
index 9e7ec2388..ded25f114 100644
--- a/apps/api/src/__tests__/snips/v2/billing.test.ts
+++ b/apps/api/src/__tests__/snips/v2/billing.test.ts
@@ -224,7 +224,7 @@ describe("Billing tests", () => {
query: "firecrawl filetype:pdf",
scrapeOptions: {
formats: ["markdown"],
- parsers: { pdf: true },
+ parsers: ["pdf"],
},
}, identity);
@@ -249,7 +249,7 @@ describe("Billing tests", () => {
query: "firecrawl filetype:pdf",
scrapeOptions: {
formats: ["markdown"],
- parsers: { pdf: false },
+ parsers: [],
},
}, identity);
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts
new file mode 100644
index 000000000..f35143ba7
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts
@@ -0,0 +1,139 @@
+import { scrape, scrapeRaw, scrapeTimeout, idmux, Identity } from "./lib";
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "parsers",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+describe("Parsers parameter tests", () => {
+ const pdfUrl = "https://www.orimi.com/pdf-test.pdf";
+ const htmlUrl = "https://firecrawl.dev";
+
+ describe("Array format", () => {
+ it.concurrent("accepts parsers: ['pdf'] and parses PDF", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: ["pdf"],
+ timeout: scrapeTimeout * 2,
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ }, scrapeTimeout * 2);
+
+ it.concurrent("accepts parsers: [] and returns PDF in base64", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: [],
+ timeout: scrapeTimeout * 2,
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("JVBER"); // base64
+ }, scrapeTimeout * 2);
+
+ it.concurrent("accepts parsers: ['pdf'] on HTML pages (no effect)", async () => {
+ const response = await scrape({
+ url: htmlUrl,
+ parsers: ["pdf"],
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("Firecrawl");
+ }, scrapeTimeout);
+
+ it.concurrent("accepts empty parsers array on HTML pages", async () => {
+ const response = await scrape({
+ url: htmlUrl,
+ parsers: [],
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("Firecrawl");
+ }, scrapeTimeout);
+ });
+
+
+ describe("Default behavior", () => {
+ it.concurrent("parses PDF by default when parsers not specified", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ timeout: scrapeTimeout * 2,
+ }, identity);
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("PDF Test File");
+ expect(response.metadata.numPages).toBeGreaterThan(0);
+ }, scrapeTimeout * 2);
+ });
+
+ describe("Invalid inputs", () => {
+ it.concurrent("rejects invalid parser types", async () => {
+ const raw = await scrapeRaw({
+ url: pdfUrl,
+ parsers: ["invalid-parser" as any],
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ }, scrapeTimeout);
+
+ it.concurrent("rejects non-array parsers", async () => {
+ const raw = await scrapeRaw({
+ url: pdfUrl,
+ parsers: "pdf" as any,
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ }, scrapeTimeout);
+
+ it.concurrent("rejects old object format", async () => {
+ const raw = await scrapeRaw({
+ url: pdfUrl,
+ parsers: { pdf: true } as any,
+ timeout: scrapeTimeout,
+ }, identity);
+
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.error).toBe("Bad Request");
+ }, scrapeTimeout);
+ });
+
+ describe("Billing implications", () => {
+ it.concurrent("bills correctly with parsers: ['pdf']", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: ["pdf"],
+ timeout: scrapeTimeout * 2,
+ }, identity);
+
+ // Should bill based on number of pages when PDF parsing is enabled
+ expect(response.metadata.creditsUsed).toBeGreaterThanOrEqual(response.metadata.numPages || 1);
+ }, scrapeTimeout * 2);
+
+ it.concurrent("bills flat rate with parsers: []", async () => {
+ const response = await scrape({
+ url: pdfUrl,
+ parsers: [],
+ timeout: scrapeTimeout * 2,
+ }, identity);
+
+ // Should bill flat rate (1 credit) when PDF parsing is disabled
+ expect(response.metadata.creditsUsed).toBe(1);
+ }, scrapeTimeout * 2);
+ });
+});
\ No newline at end of file
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 19297756a..a080fae13 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -209,9 +209,7 @@ export const screenshotFormatWithOptions = z.object({
export type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;
-export const parsersSchema = z.object({
- pdf: z.boolean().default(true),
-}).default({ pdf: true });
+export const parsersSchema = z.array(z.enum(["pdf"])).default(["pdf"]);
export type Parsers = z.infer<typeof parsersSchema>;
@@ -1094,7 +1092,7 @@ export function fromV0ScrapeOptions(
: pageOptions.removeTags,
onlyMainContent: pageOptions.onlyMainContent ?? false,
timeout: timeout,
- parsePDF: pageOptions.parsePDF,
+ parsers: pageOptions.parsePDF !== undefined ? (pageOptions.parsePDF ? ["pdf"] : []) : undefined,
actions: pageOptions.actions,
location: pageOptions.geolocation,
skipTlsVerification: pageOptions.skipTlsVerification,
@@ -1190,7 +1188,7 @@ export function fromV1ScrapeOptions(
return x;
}
}).filter(x => x !== null),
- parsers: v1ScrapeOptions.parsePDF !== undefined ? { pdf: v1ScrapeOptions.parsePDF } : undefined,
+ parsers: v1ScrapeOptions.parsePDF !== undefined ? (v1ScrapeOptions.parsePDF ? ["pdf"] : []) : undefined,
}),
internalOptions: {
teamId,
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 4c93de040..925d967e0 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -32,7 +32,7 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, interna
creditsToBeBilled += (flags?.zdrCost ?? 1);
}
- const shouldParsePDF = options.parsers?.pdf ?? true;
+ const shouldParsePDF = options.parsers?.includes("pdf") ?? true;
if (shouldParsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {
creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 87b1a2217..c4e49ea47 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -175,7 +175,7 @@ export async function scrapePDF(
): Promise<EngineScrapeResult> {
const startTime = Date.now();
- const shouldParsePDF = meta.options.parsers?.pdf ?? true;
+ const shouldParsePDF = meta.options.parsers?.includes("pdf") ?? true;
if (!shouldParsePDF) {
if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {