Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/tests): add webhook tests + refactor batch scrape lib (#1630)
Gergő Móricz077c5dd8ec6b
diff --git a/.github/workflows/test-server-self-host.yml b/.github/workflows/test-server-self-host.ymlindex ae46f29ce..76a786b3e 100644--- a/.github/workflows/test-server-self-host.yml+++ b/.github/workflows/test-server-self-host.yml@@ -114,7 +114,7 @@ jobs:env:PORT: 3003- name: Wait for server- run: pnpx wait-on tcp:3002 -t 15s+ run: pnpx wait-on tcp:3002 -t 60s- name: Wait for playwrightif: matrix.engine == 'playwright'run: pnpx wait-on tcp:3003 -t 15sdiff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex ca1862876..f76b558bd 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -108,7 +108,7 @@ jobs:working-directory: ./apps/apiid: start_index_worker- name: Wait for API- run: pnpx wait-on tcp:3002 -t 30s+ run: pnpx wait-on tcp:3002 -t 60s- name: Run snippet testsrun: |npm run test:snipsdiff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.tsindex 8d4c39468..113cf651d 100644--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts@@ -6,8 +6,8 @@ describe("Batch scrape tests", () => {urls: ["http://firecrawl.dev"]});- expect(response.body.data[0]).toHaveProperty("markdown");- expect(response.body.data[0].markdown).toContain("Firecrawl");+ expect(response.data[0]).toHaveProperty("markdown");+ expect(response.data[0].markdown).toContain("Firecrawl");}, 180000);if (!process.env.TEST_SUITE_SELF_HOSTED) {@@ -36,15 +36,15 @@ describe("Batch scrape tests", () => {},});- expect(response.body.data[0]).toHaveProperty("json");- expect(response.body.data[0].json).toHaveProperty("company_mission");- expect(typeof response.body.data[0].json.company_mission).toBe("string");- expect(response.body.data[0].json).toHaveProperty("supports_sso");- expect(response.body.data[0].json.supports_sso).toBe(false);- expect(typeof response.body.data[0].json.supports_sso).toBe("boolean");- expect(response.body.data[0].json).toHaveProperty("is_open_source");- expect(response.body.data[0].json.is_open_source).toBe(true);- expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");+ expect(response.data[0]).toHaveProperty("json");+ expect(response.data[0].json).toHaveProperty("company_mission");+ expect(typeof response.data[0].json.company_mission).toBe("string");+ expect(response.data[0].json).toHaveProperty("supports_sso");+ expect(response.data[0].json.supports_sso).toBe(false);+ expect(typeof response.data[0].json.supports_sso).toBe("boolean");+ expect(response.data[0].json).toHaveProperty("is_open_source");+ expect(response.data[0].json.is_open_source).toBe(true);+ expect(typeof response.data[0].json.is_open_source).toBe("boolean");}, 180000);});}@@ -54,6 +54,6 @@ describe("Batch scrape tests", () => {urls: ["https://firecrawl.dev/?pagewanted=all&et_blog"],});- expect(response.body.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");+ expect(response.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");}, 35000);});diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex c8d818384..d851ca821 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -1,7 +1,7 @@import { configDotenv } from "dotenv";configDotenv();-import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse } from "../../controllers/v1/types";+import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, ErrorResponse } from "../../controllers/v1/types";import request from "supertest";// =========================================@@ -106,7 +106,7 @@ function expectCrawlToSucceed(response: Awaited<ReturnType<typeof crawlStatus>>)expect(response.body.data.length).toBeGreaterThan(0);}-export async function crawl(body: CrawlRequestInput): Promise<CrawlStatusResponse> {+export async function crawl(body: CrawlRequestInput): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {const cs = await crawlStart(body);expectCrawlStartToSucceed(cs);@@ -141,7 +141,7 @@ async function batchScrapeStatus(id: string) {.send();}-function expectBatchScrapeStartToSucceed(response: Awaited<ReturnType<typeof batchScrape>>) {+function expectBatchScrapeStartToSucceed(response: Awaited<ReturnType<typeof batchScrapeStart>>) {expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);expect(typeof response.body.id).toBe("string");@@ -157,7 +157,7 @@ function expectBatchScrapeToSucceed(response: Awaited<ReturnType<typeof batchScrexpect(response.body.data.length).toBeGreaterThan(0);}-export async function batchScrape(body: BatchScrapeRequestInput): ReturnType<typeof batchScrapeStatus> {+export async function batchScrape(body: BatchScrapeRequestInput): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {const bss = await batchScrapeStart(body);expectBatchScrapeStartToSucceed(bss);@@ -170,7 +170,7 @@ export async function batchScrape(body: BatchScrapeRequestInput): ReturnType<typ} while (x.body.status === "scraping");expectBatchScrapeToSucceed(x);- return x;+ return x.body;}// =========================================diff --git a/apps/api/src/__tests__/snips/webhook.test.ts b/apps/api/src/__tests__/snips/webhook.test.tsnew file mode 100644index 000000000..86a5db258--- /dev/null+++ b/apps/api/src/__tests__/snips/webhook.test.ts@@ -0,0 +1,118 @@+import { crawl, batchScrape } from "./lib";+import Express from "express";+import bodyParser from "body-parser";+import type { WebhookEventType } from "src/types";+import type { Document } from "src/controllers/v1/types";++const WEBHOOK_PORT_CRAWL = 3008;+const WEBHOOK_PORT_BATCH_SCRAPE = 3009;++describe("Webhook tests", () => {+ it.concurrent("webhook works properly for crawl", async () => {+ const app = Express();+ app.use(bodyParser.json());++ const calls: {+ type: WebhookEventType,+ id: string,+ data: any;+ }[] = [];++ app.post("/webhook", (req, res) => {+ calls.push({+ type: req.body.type,+ id: req.body.id,+ data: req.body.data,+ });+ res.json({ ok: true })+ });++ const server = app.listen(WEBHOOK_PORT_CRAWL);++ const res = await crawl({+ url: "https://firecrawl.dev",+ limit: 10,+ webhook: {+ url: `http://localhost:${WEBHOOK_PORT_CRAWL}/webhook`,+ },+ });++ server.close();++ let hasStarted = false;+ let hasCompleted = false;++ for (const call of calls) {+ expect(call.type).toMatch(/^crawl\./);++ if (call.type === "crawl.started") {+ hasStarted = true;+ } else if (call.type === "crawl.page") {+ const page = call.data[0] as Document;+ expect(res.data.find(x => page.metadata.scrapeId !== undefined && page.metadata.scrapeId === x.metadata.scrapeId)).toBeDefined();+ } else if (call.type === "crawl.completed") {+ hasCompleted = true;+ }+ }++ expect(hasStarted).toBe(true);+ expect(hasCompleted).toBe(true);++ expect(res.data.length).toBe(calls.filter(x => x.type === "crawl.page").length);+ }, 600000);++ it.concurrent("webhook works properly for batch scrape", async () => {+ const app = Express();+ app.use(bodyParser.json());++ const calls: {+ type: WebhookEventType,+ id: string,+ data: any;+ }[] = [];++ app.post("/webhook", (req, res) => {+ calls.push({+ type: req.body.type,+ id: req.body.id,+ data: req.body.data,+ });+ res.json({ ok: true })+ });++ const server = app.listen(WEBHOOK_PORT_BATCH_SCRAPE);++ const res = await batchScrape({+ urls: [+ "https://firecrawl.dev",+ "https://firecrawl.dev/blog",+ ],+ webhook: {+ url: `http://localhost:${WEBHOOK_PORT_BATCH_SCRAPE}/webhook`,+ },+ });++ server.close();++ let hasStarted = false;+ let hasCompleted = false;++ for (const call of calls) {+ expect(call.type).toMatch(/^batch_scrape\./);++ if (call.type === "batch_scrape.started") {+ hasStarted = true;+ } else if (call.type === "batch_scrape.page") {+ const page = call.data[0] as Document;+ expect(res.data.find(x => page.metadata.scrapeId !== undefined && page.metadata.scrapeId === x.metadata.scrapeId)).toBeDefined();+ } else if (call.type === "batch_scrape.completed") {+ hasCompleted = true;+ }+ }++ expect(hasStarted).toBe(true);+ expect(hasCompleted).toBe(true);++ expect(res.data.length).toBe(calls.filter(x => x.type === "batch_scrape.page").length);+ }, 600000);+});\ No newline at end of file