Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
ENG-3088: Change parsers parameter from object to array format (#1931)
Gergő Móriczcb1415aa203f
diff --git a/apps/api/src/__tests__/snips/v2/billing.test.ts b/apps/api/src/__tests__/snips/v2/billing.test.tsindex 9e7ec2388..ded25f114 100644--- a/apps/api/src/__tests__/snips/v2/billing.test.ts+++ b/apps/api/src/__tests__/snips/v2/billing.test.ts@@ -224,7 +224,7 @@ describe("Billing tests", () => {query: "firecrawl filetype:pdf",scrapeOptions: {formats: ["markdown"],- parsers: { pdf: true },+ parsers: ["pdf"],},}, identity);@@ -249,7 +249,7 @@ describe("Billing tests", () => {query: "firecrawl filetype:pdf",scrapeOptions: {formats: ["markdown"],- parsers: { pdf: false },+ parsers: [],},}, identity);diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.tsnew file mode 100644index 000000000..f35143ba7--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts@@ -0,0 +1,139 @@+import { scrape, scrapeRaw, scrapeTimeout, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "parsers",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++describe("Parsers parameter tests", () => {+ const pdfUrl = "https://www.orimi.com/pdf-test.pdf";+ const htmlUrl = "https://firecrawl.dev";++ describe("Array format", () => {+ it.concurrent("accepts parsers: ['pdf'] and parses PDF", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: ["pdf"],+ timeout: scrapeTimeout * 2,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBeGreaterThan(0);+ }, scrapeTimeout * 2);++ it.concurrent("accepts parsers: [] and returns PDF in base64", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: [],+ timeout: scrapeTimeout * 2,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("JVBER"); // base64+ }, scrapeTimeout * 2);++ it.concurrent("accepts parsers: ['pdf'] on HTML pages (no effect)", async () => {+ const response = await scrape({+ url: htmlUrl,+ parsers: ["pdf"],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("Firecrawl");+ }, scrapeTimeout);++ it.concurrent("accepts empty parsers array on HTML pages", async () => {+ const response = await scrape({+ url: htmlUrl,+ parsers: [],+ timeout: scrapeTimeout,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("Firecrawl");+ }, scrapeTimeout);+ });+++ describe("Default behavior", () => {+ it.concurrent("parses PDF by default when parsers not specified", async () => {+ const response = await scrape({+ url: pdfUrl,+ timeout: scrapeTimeout * 2,+ }, identity);++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("PDF Test File");+ expect(response.metadata.numPages).toBeGreaterThan(0);+ }, scrapeTimeout * 2);+ });++ describe("Invalid inputs", () => {+ it.concurrent("rejects invalid parser types", async () => {+ const raw = await scrapeRaw({+ url: pdfUrl,+ parsers: ["invalid-parser" as any],+ timeout: scrapeTimeout,+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ }, scrapeTimeout);++ it.concurrent("rejects non-array parsers", async () => {+ const raw = await scrapeRaw({+ url: pdfUrl,+ parsers: "pdf" as any,+ timeout: scrapeTimeout,+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ }, scrapeTimeout);++ it.concurrent("rejects old object format", async () => {+ const raw = await scrapeRaw({+ url: pdfUrl,+ parsers: { pdf: true } as any,+ timeout: scrapeTimeout,+ }, identity);++ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.error).toBe("Bad Request");+ }, scrapeTimeout);+ });++ describe("Billing implications", () => {+ it.concurrent("bills correctly with parsers: ['pdf']", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: ["pdf"],+ timeout: scrapeTimeout * 2,+ }, identity);++ // Should bill based on number of pages when PDF parsing is enabled+ expect(response.metadata.creditsUsed).toBeGreaterThanOrEqual(response.metadata.numPages || 1);+ }, scrapeTimeout * 2);++ it.concurrent("bills flat rate with parsers: []", async () => {+ const response = await scrape({+ url: pdfUrl,+ parsers: [],+ timeout: scrapeTimeout * 2,+ }, identity);++ // Should bill flat rate (1 credit) when PDF parsing is disabled+ expect(response.metadata.creditsUsed).toBe(1);+ }, scrapeTimeout * 2);+ });+});\ No newline at end of filediff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 19297756a..a080fae13 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -209,9 +209,7 @@ export const screenshotFormatWithOptions = z.object({export type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;-export const parsersSchema = z.object({- pdf: z.boolean().default(true),-}).default({ pdf: true });+export const parsersSchema = z.array(z.enum(["pdf"])).default(["pdf"]);export type Parsers = z.infer<typeof parsersSchema>;@@ -1094,7 +1092,7 @@ export function fromV0ScrapeOptions(: pageOptions.removeTags,onlyMainContent: pageOptions.onlyMainContent ?? false,timeout: timeout,- parsePDF: pageOptions.parsePDF,+ parsers: pageOptions.parsePDF !== undefined ? (pageOptions.parsePDF ? ["pdf"] : []) : undefined,actions: pageOptions.actions,location: pageOptions.geolocation,skipTlsVerification: pageOptions.skipTlsVerification,@@ -1190,7 +1188,7 @@ export function fromV1ScrapeOptions(return x;}}).filter(x => x !== null),- parsers: v1ScrapeOptions.parsePDF !== undefined ? { pdf: v1ScrapeOptions.parsePDF } : undefined,+ parsers: v1ScrapeOptions.parsePDF !== undefined ? (v1ScrapeOptions.parsePDF ? ["pdf"] : []) : undefined,}),internalOptions: {teamId,diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 4c93de040..925d967e0 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -32,7 +32,7 @@ export async function calculateCreditsToBeBilled(options: ScrapeOptions, internacreditsToBeBilled += (flags?.zdrCost ?? 1);}- const shouldParsePDF = options.parsers?.pdf ?? true;+ const shouldParsePDF = options.parsers?.includes("pdf") ?? true;if (shouldParsePDF && document.metadata?.numPages !== undefined && document.metadata.numPages > 1) {creditsToBeBilled += creditsPerPDFPage * (document.metadata.numPages - 1);}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 87b1a2217..c4e49ea47 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -175,7 +175,7 @@ export async function scrapePDF(): Promise<EngineScrapeResult> {const startTime = Date.now();- const shouldParsePDF = meta.options.parsers?.pdf ?? true;+ const shouldParsePDF = meta.options.parsers?.includes("pdf") ?? true;if (!shouldParsePDF) {if (meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null) {