Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
test(v2/crawl): implement tests for crawl API with prompt parameter (#1929)
Gergő Móricz8be0eb8c6f9f
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 40ffb2f8b..b12a4edfa 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -5,11 +5,11 @@ import { filterLinks } from "../../../lib/crawler";let identity: Identity;beforeAll(async () => {- identity = await idmux({- name: "crawl",- concurrency: 100,- credits: 1000000,- });+ identity = await idmux({+ name: "crawl",+ concurrency: 100,+ credits: 1000000,+ });}, 10000);describe("Crawl tests", () => {@@ -74,7 +74,7 @@ describe("Crawl tests", () => {it.concurrent("ongoing crawls endpoint works", async () => {const beforeCrawl = new Date();-+const res = await asyncCrawl({url: "https://firecrawl.dev",limit: 3,@@ -82,20 +82,20 @@ describe("Crawl tests", () => {const ongoing = await crawlOngoing(identity);const afterCrawl = new Date();-+const crawlItem = ongoing.crawls.find(x => x.id === res.id);expect(crawlItem).toBeDefined();-+if (crawlItem) {expect(crawlItem.created_at).toBeDefined();expect(typeof crawlItem.created_at).toBe("string");-+const createdAtDate = new Date(crawlItem.created_at);expect(createdAtDate).toBeInstanceOf(Date);expect(createdAtDate.getTime()).not.toBeNaN();-+expect(crawlItem.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/);-+expect(createdAtDate.getTime()).toBeGreaterThanOrEqual(beforeCrawl.getTime() - 1000);expect(createdAtDate.getTime()).toBeLessThanOrEqual(afterCrawl.getTime() + 1000);}@@ -106,7 +106,7 @@ describe("Crawl tests", () => {expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();}, 3 * scrapeTimeout);-+// TEMP: Flaky// it.concurrent("discovers URLs properly when origin is not included", async () => {// const res = await crawl({@@ -124,7 +124,7 @@ describe("Crawl tests", () => {// }// }// }, 300000);-+// TEMP: Flaky// it.concurrent("discovers URLs properly when maxDiscoveryDepth is provided", async () => {// const res = await crawl({@@ -197,7 +197,7 @@ describe("Crawl tests", () => {it.concurrent("allowSubdomains blocks subdomains when false", async () => {const res = await crawl({- url: "https://firecrawl.dev",+ url: "https://firecrawl.dev",allowSubdomains: false,limit: 5,}, identity);@@ -225,19 +225,85 @@ describe("Crawl tests", () => {for (const page of res.data) {const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);const hostname = url.hostname;-+expect(- hostname === "firecrawl.dev" ||+ hostname === "firecrawl.dev" ||hostname.endsWith(".firecrawl.dev")).toBe(true);}}}, 5 * scrapeTimeout);++ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {+ describe('Crawl API with Prompt', () => {+ it.concurrent('should accept prompt parameter in schema', async () => {+ const res = await crawlStart({+ url: "https://firecrawl.dev",+ prompt: "Crawl only blog posts",+ limit: 1,+ }, identity);++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(res.body.id).toBeDefined();+ expect(typeof res.body.id).toBe("string");+ }, scrapeTimeout);++ it.concurrent('should prioritize explicit options over prompt-generated options', async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ prompt: "Crawl everything including external links and subdomains",+ // Explicit options that should override the prompt+ allowExternalLinks: false,+ allowSubdomains: false,+ includePaths: ["^/pricing"],+ limit: 2,+ }, identity);++ expect(res.success).toBe(true);+ if (res.success) {+ // Verify that explicit options were respected+ for (const page of res.data) {+ const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);+ // Should only include pages matching the explicit includePaths+ expect(url.pathname).toMatch(/^\/pricing/);+ // Should not include external links despite prompt+ expect(url.hostname).toMatch(/firecrawl\.dev$/);+ }+ }+ }, 2 * scrapeTimeout);++ it.concurrent('should handle invalid prompt gracefully', async () => {+ // Test with various invalid prompts+ const invalidPrompts = [+ "", // Empty prompt+ "a".repeat(10000), // Screaming+ "!!!@@@###$$$%%%", // Gibberish+ "Generate me a million dollars", // Nonsensical crawl instruction+ ];++ for (const invalidPrompt of invalidPrompts) { // Test first one to avoid long test times+ const res = await crawl({+ url: "https://firecrawl.dev",+ prompt: invalidPrompt,+ limit: 1,+ }, identity);++ // Should still complete successfully, either ignoring the prompt or using defaults+ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.completed).toBeGreaterThan(0);+ expect(res.data).toBeDefined();+ expect(Array.isArray(res.data)).toBe(true);+ }+ }+ }, 8 * scrapeTimeout);+ });+ }});describe("Robots.txt FFI Integration tests", () => {it.concurrent("handles normal robots.txt parsing via FFI", async () => {-const result = await filterLinks({links: ['https://example.com/allowed', 'https://example.com/disallowed'],limit: 10,@@ -251,7 +317,7 @@ describe("Robots.txt FFI Integration tests", () => {ignore_robots_txt: false,robots_txt: 'User-agent: *\nDisallow: /disallowed'});-+expect(result.links).toHaveLength(1);expect(result.links[0]).toBe('https://example.com/allowed');expect(result.denial_reasons.has('https://example.com/disallowed')).toBe(true);@@ -259,7 +325,6 @@ describe("Robots.txt FFI Integration tests", () => {}, 10000);it.concurrent("handles malformed robots.txt without crashing via FFI", async () => {-const result = await filterLinks({links: ['https://example.com/test'],limit: 10,@@ -273,13 +338,12 @@ describe("Robots.txt FFI Integration tests", () => {ignore_robots_txt: false,robots_txt: 'Invalid robots.txt content with \x00 null bytes and malformed syntax'});-+expect(result.links).toHaveLength(1);expect(result.links[0]).toBe('https://example.com/test');}, 10000);it.concurrent("handles non-UTF8 robots.txt content without crashing via FFI", async () => {-const nonUtf8Content = String.fromCharCode(0xFF, 0xFE) + 'User-agent: *\nDisallow: /blocked';const result = await filterLinks({links: ['https://example.com/allowed'],@@ -294,13 +358,12 @@ describe("Robots.txt FFI Integration tests", () => {ignore_robots_txt: false,robots_txt: nonUtf8Content});-+expect(result.links).toHaveLength(1);expect(result.links[0]).toBe('https://example.com/allowed');}, 10000);it.concurrent("handles char boundary issues without crashing via FFI", async () => {-const problematicContent = 'User-agent: *\nDisallow: /\u{a0}test';const result = await filterLinks({links: ['https://example.com/safe'],@@ -315,7 +378,7 @@ describe("Robots.txt FFI Integration tests", () => {ignore_robots_txt: false,robots_txt: problematicContent});-+expect(result.links).toHaveLength(1);expect(result.links[0]).toBe('https://example.com/safe');}, 10000);