Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/tests): add webhook tests + refactor batch scrape lib (#1630)

Gergő Móricz077c5dd8ec6b
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.yml
index ae46f29ce..76a786b3e 100644
--- a/.github/workflows/test-server-self-host.yml
+++ b/.github/workflows/test-server-self-host.yml
@@ -114,7 +114,7 @@ jobs:
env:
PORT: 3003
- name: Wait for server
- run: pnpx wait-on tcp:3002 -t 15s
+ run: pnpx wait-on tcp:3002 -t 60s
- name: Wait for playwright
if: matrix.engine == 'playwright'
run: pnpx wait-on tcp:3003 -t 15s
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index ca1862876..f76b558bd 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -108,7 +108,7 @@ jobs:
working-directory: ./apps/api
id: start_index_worker
- name: Wait for API
- run: pnpx wait-on tcp:3002 -t 30s
+ run: pnpx wait-on tcp:3002 -t 60s
- name: Run snippet tests
run: |
npm run test:snips
diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.ts
index 8d4c39468..113cf651d 100644
--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts
+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts
@@ -6,8 +6,8 @@ describe("Batch scrape tests", () => {
urls: ["http://firecrawl.dev"]
});
- expect(response.body.data[0]).toHaveProperty("markdown");
- expect(response.body.data[0].markdown).toContain("Firecrawl");
+ expect(response.data[0]).toHaveProperty("markdown");
+ expect(response.data[0].markdown).toContain("Firecrawl");
}, 180000);
if (!process.env.TEST_SUITE_SELF_HOSTED) {
@@ -36,15 +36,15 @@ describe("Batch scrape tests", () => {
},
});
- expect(response.body.data[0]).toHaveProperty("json");
- expect(response.body.data[0].json).toHaveProperty("company_mission");
- expect(typeof response.body.data[0].json.company_mission).toBe("string");
- expect(response.body.data[0].json).toHaveProperty("supports_sso");
- expect(response.body.data[0].json.supports_sso).toBe(false);
- expect(typeof response.body.data[0].json.supports_sso).toBe("boolean");
- expect(response.body.data[0].json).toHaveProperty("is_open_source");
- expect(response.body.data[0].json.is_open_source).toBe(true);
- expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");
+ expect(response.data[0]).toHaveProperty("json");
+ expect(response.data[0].json).toHaveProperty("company_mission");
+ expect(typeof response.data[0].json.company_mission).toBe("string");
+ expect(response.data[0].json).toHaveProperty("supports_sso");
+ expect(response.data[0].json.supports_sso).toBe(false);
+ expect(typeof response.data[0].json.supports_sso).toBe("boolean");
+ expect(response.data[0].json).toHaveProperty("is_open_source");
+ expect(response.data[0].json.is_open_source).toBe(true);
+ expect(typeof response.data[0].json.is_open_source).toBe("boolean");
}, 180000);
});
}
@@ -54,6 +54,6 @@ describe("Batch scrape tests", () => {
urls: ["https://firecrawl.dev/?pagewanted=all&et_blog"],
});
- expect(response.body.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");
+ expect(response.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");
}, 35000);
});
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index c8d818384..d851ca821 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -1,7 +1,7 @@
import { configDotenv } from "dotenv";
configDotenv();
-import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse } from "../../controllers/v1/types";
+import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, ErrorResponse } from "../../controllers/v1/types";
import request from "supertest";
// =========================================
@@ -106,7 +106,7 @@ function expectCrawlToSucceed(response: Awaited<ReturnType<typeof crawlStatus>>)
expect(response.body.data.length).toBeGreaterThan(0);
}
-export async function crawl(body: CrawlRequestInput): Promise<CrawlStatusResponse> {
+export async function crawl(body: CrawlRequestInput): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {
const cs = await crawlStart(body);
expectCrawlStartToSucceed(cs);
@@ -141,7 +141,7 @@ async function batchScrapeStatus(id: string) {
.send();
}
-function expectBatchScrapeStartToSucceed(response: Awaited<ReturnType<typeof batchScrape>>) {
+function expectBatchScrapeStartToSucceed(response: Awaited<ReturnType<typeof batchScrapeStart>>) {
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
expect(typeof response.body.id).toBe("string");
@@ -157,7 +157,7 @@ function expectBatchScrapeToSucceed(response: Awaited<ReturnType<typeof batchScr
expect(response.body.data.length).toBeGreaterThan(0);
}
-export async function batchScrape(body: BatchScrapeRequestInput): ReturnType<typeof batchScrapeStatus> {
+export async function batchScrape(body: BatchScrapeRequestInput): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {
const bss = await batchScrapeStart(body);
expectBatchScrapeStartToSucceed(bss);
@@ -170,7 +170,7 @@ export async function batchScrape(body: BatchScrapeRequestInput): ReturnType<typ
} while (x.body.status === "scraping");
expectBatchScrapeToSucceed(x);
- return x;
+ return x.body;
}
// =========================================
diff --git a/apps/api/src/__tests__/snips/webhook.test.ts b/apps/api/src/__tests__/snips/webhook.test.ts
new file mode 100644
index 000000000..86a5db258
--- /dev/null
+++ b/apps/api/src/__tests__/snips/webhook.test.ts
@@ -0,0 +1,118 @@
+import { crawl, batchScrape } from "./lib";
+import Express from "express";
+import bodyParser from "body-parser";
+import type { WebhookEventType } from "src/types";
+import type { Document } from "src/controllers/v1/types";
+
+const WEBHOOK_PORT_CRAWL = 3008;
+const WEBHOOK_PORT_BATCH_SCRAPE = 3009;
+
+describe("Webhook tests", () => {
+ it.concurrent("webhook works properly for crawl", async () => {
+ const app = Express();
+ app.use(bodyParser.json());
+
+ const calls: {
+ type: WebhookEventType,
+ id: string,
+ data: any;
+ }[] = [];
+
+ app.post("/webhook", (req, res) => {
+ calls.push({
+ type: req.body.type,
+ id: req.body.id,
+ data: req.body.data,
+ });
+ res.json({ ok: true })
+ });
+
+ const server = app.listen(WEBHOOK_PORT_CRAWL);
+
+ const res = await crawl({
+ url: "https://firecrawl.dev",
+ limit: 10,
+ webhook: {
+ url: `http://localhost:${WEBHOOK_PORT_CRAWL}/webhook`,
+ },
+ });
+
+ server.close();
+
+ let hasStarted = false;
+ let hasCompleted = false;
+
+ for (const call of calls) {
+ expect(call.type).toMatch(/^crawl\./);
+
+ if (call.type === "crawl.started") {
+ hasStarted = true;
+ } else if (call.type === "crawl.page") {
+ const page = call.data[0] as Document;
+ expect(res.data.find(x => page.metadata.scrapeId !== undefined && page.metadata.scrapeId === x.metadata.scrapeId)).toBeDefined();
+ } else if (call.type === "crawl.completed") {
+ hasCompleted = true;
+ }
+ }
+
+ expect(hasStarted).toBe(true);
+ expect(hasCompleted).toBe(true);
+
+ expect(res.data.length).toBe(calls.filter(x => x.type === "crawl.page").length);
+ }, 600000);
+
+ it.concurrent("webhook works properly for batch scrape", async () => {
+ const app = Express();
+ app.use(bodyParser.json());
+
+ const calls: {
+ type: WebhookEventType,
+ id: string,
+ data: any;
+ }[] = [];
+
+ app.post("/webhook", (req, res) => {
+ calls.push({
+ type: req.body.type,
+ id: req.body.id,
+ data: req.body.data,
+ });
+ res.json({ ok: true })
+ });
+
+ const server = app.listen(WEBHOOK_PORT_BATCH_SCRAPE);
+
+ const res = await batchScrape({
+ urls: [
+ "https://firecrawl.dev",
+ "https://firecrawl.dev/blog",
+ ],
+ webhook: {
+ url: `http://localhost:${WEBHOOK_PORT_BATCH_SCRAPE}/webhook`,
+ },
+ });
+
+ server.close();
+
+ let hasStarted = false;
+ let hasCompleted = false;
+
+ for (const call of calls) {
+ expect(call.type).toMatch(/^batch_scrape\./);
+
+ if (call.type === "batch_scrape.started") {
+ hasStarted = true;
+ } else if (call.type === "batch_scrape.page") {
+ const page = call.data[0] as Document;
+ expect(res.data.find(x => page.metadata.scrapeId !== undefined && page.metadata.scrapeId === x.metadata.scrapeId)).toBeDefined();
+ } else if (call.type === "batch_scrape.completed") {
+ hasCompleted = true;
+ }
+ }
+
+ expect(hasStarted).toBe(true);
+ expect(hasCompleted).toBe(true);
+
+ expect(res.data.length).toBe(calls.filter(x => x.type === "batch_scrape.page").length);
+ }, 600000);
+});
\ No newline at end of file