Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

test(v2/crawl): implement tests for crawl API with prompt parameter (#1929)

Gergő Móricz8be0eb8c6f9f
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 40ffb2f8b..b12a4edfa 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -5,11 +5,11 @@ import { filterLinks } from "../../../lib/crawler";
let identity: Identity;
beforeAll(async () => {
- identity = await idmux({
- name: "crawl",
- concurrency: 100,
- credits: 1000000,
- });
+ identity = await idmux({
+ name: "crawl",
+ concurrency: 100,
+ credits: 1000000,
+ });
}, 10000);
describe("Crawl tests", () => {
@@ -74,7 +74,7 @@ describe("Crawl tests", () => {
it.concurrent("ongoing crawls endpoint works", async () => {
const beforeCrawl = new Date();
-
+
const res = await asyncCrawl({
url: "https://firecrawl.dev",
limit: 3,
@@ -82,20 +82,20 @@ describe("Crawl tests", () => {
const ongoing = await crawlOngoing(identity);
const afterCrawl = new Date();
-
+
const crawlItem = ongoing.crawls.find(x => x.id === res.id);
expect(crawlItem).toBeDefined();
-
+
if (crawlItem) {
expect(crawlItem.created_at).toBeDefined();
expect(typeof crawlItem.created_at).toBe("string");
-
+
const createdAtDate = new Date(crawlItem.created_at);
expect(createdAtDate).toBeInstanceOf(Date);
expect(createdAtDate.getTime()).not.toBeNaN();
-
+
expect(crawlItem.created_at).toMatch(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/);
-
+
expect(createdAtDate.getTime()).toBeGreaterThanOrEqual(beforeCrawl.getTime() - 1000);
expect(createdAtDate.getTime()).toBeLessThanOrEqual(afterCrawl.getTime() + 1000);
}
@@ -106,7 +106,7 @@ describe("Crawl tests", () => {
expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();
}, 3 * scrapeTimeout);
-
+
// TEMP: Flaky
// it.concurrent("discovers URLs properly when origin is not included", async () => {
// const res = await crawl({
@@ -124,7 +124,7 @@ describe("Crawl tests", () => {
// }
// }
// }, 300000);
-
+
// TEMP: Flaky
// it.concurrent("discovers URLs properly when maxDiscoveryDepth is provided", async () => {
// const res = await crawl({
@@ -197,7 +197,7 @@ describe("Crawl tests", () => {
it.concurrent("allowSubdomains blocks subdomains when false", async () => {
const res = await crawl({
- url: "https://firecrawl.dev",
+ url: "https://firecrawl.dev",
allowSubdomains: false,
limit: 5,
}, identity);
@@ -225,19 +225,85 @@ describe("Crawl tests", () => {
for (const page of res.data) {
const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);
const hostname = url.hostname;
-
+
expect(
- hostname === "firecrawl.dev" ||
+ hostname === "firecrawl.dev" ||
hostname.endsWith(".firecrawl.dev")
).toBe(true);
}
}
}, 5 * scrapeTimeout);
+
+ if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {
+ describe('Crawl API with Prompt', () => {
+ it.concurrent('should accept prompt parameter in schema', async () => {
+ const res = await crawlStart({
+ url: "https://firecrawl.dev",
+ prompt: "Crawl only blog posts",
+ limit: 1,
+ }, identity);
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ expect(res.body.id).toBeDefined();
+ expect(typeof res.body.id).toBe("string");
+ }, scrapeTimeout);
+
+ it.concurrent('should prioritize explicit options over prompt-generated options', async () => {
+ const res = await crawl({
+ url: "https://firecrawl.dev",
+ prompt: "Crawl everything including external links and subdomains",
+ // Explicit options that should override the prompt
+ allowExternalLinks: false,
+ allowSubdomains: false,
+ includePaths: ["^/pricing"],
+ limit: 2,
+ }, identity);
+
+ expect(res.success).toBe(true);
+ if (res.success) {
+ // Verify that explicit options were respected
+ for (const page of res.data) {
+ const url = new URL(page.metadata.url ?? page.metadata.sourceURL!);
+ // Should only include pages matching the explicit includePaths
+ expect(url.pathname).toMatch(/^\/pricing/);
+ // Should not include external links despite prompt
+ expect(url.hostname).toMatch(/firecrawl\.dev$/);
+ }
+ }
+ }, 2 * scrapeTimeout);
+
+ it.concurrent('should handle invalid prompt gracefully', async () => {
+ // Test with various invalid prompts
+ const invalidPrompts = [
+ "", // Empty prompt
+ "a".repeat(10000), // Screaming
+ "!!!@@@###$$$%%%", // Gibberish
+ "Generate me a million dollars", // Nonsensical crawl instruction
+ ];
+
+ for (const invalidPrompt of invalidPrompts) { // Test first one to avoid long test times
+ const res = await crawl({
+ url: "https://firecrawl.dev",
+ prompt: invalidPrompt,
+ limit: 1,
+ }, identity);
+
+ // Should still complete successfully, either ignoring the prompt or using defaults
+ expect(res.success).toBe(true);
+ if (res.success) {
+ expect(res.completed).toBeGreaterThan(0);
+ expect(res.data).toBeDefined();
+ expect(Array.isArray(res.data)).toBe(true);
+ }
+ }
+ }, 8 * scrapeTimeout);
+ });
+ }
});
describe("Robots.txt FFI Integration tests", () => {
it.concurrent("handles normal robots.txt parsing via FFI", async () => {
-
const result = await filterLinks({
links: ['https://example.com/allowed', 'https://example.com/disallowed'],
limit: 10,
@@ -251,7 +317,7 @@ describe("Robots.txt FFI Integration tests", () => {
ignore_robots_txt: false,
robots_txt: 'User-agent: *\nDisallow: /disallowed'
});
-
+
expect(result.links).toHaveLength(1);
expect(result.links[0]).toBe('https://example.com/allowed');
expect(result.denial_reasons.has('https://example.com/disallowed')).toBe(true);
@@ -259,7 +325,6 @@ describe("Robots.txt FFI Integration tests", () => {
}, 10000);
it.concurrent("handles malformed robots.txt without crashing via FFI", async () => {
-
const result = await filterLinks({
links: ['https://example.com/test'],
limit: 10,
@@ -273,13 +338,12 @@ describe("Robots.txt FFI Integration tests", () => {
ignore_robots_txt: false,
robots_txt: 'Invalid robots.txt content with \x00 null bytes and malformed syntax'
});
-
+
expect(result.links).toHaveLength(1);
expect(result.links[0]).toBe('https://example.com/test');
}, 10000);
it.concurrent("handles non-UTF8 robots.txt content without crashing via FFI", async () => {
-
const nonUtf8Content = String.fromCharCode(0xFF, 0xFE) + 'User-agent: *\nDisallow: /blocked';
const result = await filterLinks({
links: ['https://example.com/allowed'],
@@ -294,13 +358,12 @@ describe("Robots.txt FFI Integration tests", () => {
ignore_robots_txt: false,
robots_txt: nonUtf8Content
});
-
+
expect(result.links).toHaveLength(1);
expect(result.links[0]).toBe('https://example.com/allowed');
}, 10000);
it.concurrent("handles char boundary issues without crashing via FFI", async () => {
-
const problematicContent = 'User-agent: *\nDisallow: /\u{a0}test';
const result = await filterLinks({
links: ['https://example.com/safe'],
@@ -315,7 +378,7 @@ describe("Robots.txt FFI Integration tests", () => {
ignore_robots_txt: false,
robots_txt: problematicContent
});
-
+
expect(result.links).toHaveLength(1);
expect(result.links[0]).toBe('https://example.com/safe');
}, 10000);