Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): CI (#1692)
GergΕ MΓ³ricz9174e0c8a0c6
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex 8903458a3..6690926bc 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -41,7 +41,7 @@ describe("Crawl tests", () => {}}, 120000);- it.concurrent("delay parameter works", async () => {+ it.only("delay parameter works", async () => {await crawl({url: "https://firecrawl.dev",limit: 3,diff --git a/apps/api/src/__tests__/snips/extract.test.ts b/apps/api/src/__tests__/snips/extract.test.tsindex acba310d0..b305d3be6 100644--- a/apps/api/src/__tests__/snips/extract.test.ts+++ b/apps/api/src/__tests__/snips/extract.test.ts@@ -20,6 +20,9 @@ describe("Extract tests", () => {"is_open_source"]},+ scrapeOptions: {+ timeout: 75000,+ },origin: "api-sdk",});@@ -28,7 +31,7 @@ describe("Extract tests", () => {expect(res.data).toHaveProperty("is_open_source");expect(typeof res.data.is_open_source).toBe("boolean");expect(res.data.is_open_source).toBe(true);- }, 60000);+ }, 90000);it.concurrent("works with unsupported JSON schema parameters", async () => {const res = await extract({@@ -45,12 +48,15 @@ describe("Extract tests", () => {"company_name"]},+ scrapeOptions: {+ timeout: 75000,+ },origin: "api-sdk",});expect(res.data).toHaveProperty("company_name");expect(typeof res.data.company_name).toBe("string")- }, 60000);+ }, 90000);} else {it.concurrent("dummy test", () => {expect(true).toBe(true);diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex c5479d199..a21d9f312 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -1,7 +1,7 @@import { configDotenv } from "dotenv";configDotenv();-import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse } from "../../controllers/v1/types";+import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse, CrawlErrorsResponse } from "../../controllers/v1/types";import request from "supertest";// =========================================@@ -147,6 +147,18 @@ export async function asyncCrawlWaitForFinish(id: string, identity = defaultIdenreturn x.body;}+export async function crawlErrors(id: string, identity = defaultIdentity): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {+ const res = await request(TEST_URL)+ .get("/v1/crawl/" + id + "/errors")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .send();++ expect(res.statusCode).toBe(200);+ expect(res.body.success).not.toBe(false);++ return res.body;+}+export async function crawl(body: CrawlRequestInput, identity = defaultIdentity): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {const cs = await crawlStart(body, identity);expectCrawlStartToSucceed(cs);@@ -159,6 +171,11 @@ export async function crawl(body: CrawlRequestInput, identity = defaultIdentity)expect(typeof x.body.status).toBe("string");} while (x.body.status === "scraping");+ const errors = await crawlErrors(cs.body.id, identity);+ if (errors.errors.length > 0) {+ console.warn("Crawl ", cs.body.id, " had errors:", errors.errors);+ }+expectCrawlToSucceed(x);return x.body;}diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 27ea6b2a7..f8066a655 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -1,6 +1,10 @@import { scrape, scrapeStatus, scrapeWithFailure } from "./lib";import crypto from "crypto";+// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests+const scrapeTimeout = 75000;+const indexCooldown = 30000;+describe("Scrape tests", () => {it.concurrent("mocking works properly", async () => {// depends on falsified mock mocking-works-properly@@ -10,51 +14,51 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",useMock: "mocking-works-properly",- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown).toBe("this is fake data coming from the mocking system!",);- }, 60000);+ }, scrapeTimeout);it.concurrent("works", async () => {const response = await scrape({url: "http://firecrawl.dev",- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown).toContain("Firecrawl");- }, 60000);+ }, scrapeTimeout);it.concurrent("handles non-UTF-8 encodings", async () => {const response = await scrape({url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown).toContain("γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ");- }, 60000);+ }, scrapeTimeout);if (process.env.TEST_SUITE_SELF_HOSTED && process.env.PROXY_SERVER) {it.concurrent("self-hosted proxy works", async () => {const response = await scrape({url: "https://icanhazip.com",- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);- }, 70000);+ }, scrapeTimeout);it.concurrent("self-hosted proxy works on playwright", async () => {const response = await scrape({url: "https://icanhazip.com",waitFor: 100,- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);- }, 70000);+ }, scrapeTimeout);}if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.PLAYWRIGHT_MICROSERVICE_URL) {@@ -62,11 +66,11 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",waitFor: 2000,- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown).toContain("Firecrawl");- }, 60000);+ }, scrapeTimeout);}describe("JSON scrape support", () => {@@ -74,19 +78,19 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["rawHtml"],- timeout: 60000,+ timeout: scrapeTimeout,});const obj = JSON.parse(response.rawHtml!);expect(obj.id).toBe(1);- }, 70000);+ }, scrapeTimeout);});if (!process.env.TEST_SUITE_SELF_HOSTED) {it.concurrent("scrape status works", async () => {const response = await scrape({url: "http://firecrawl.dev",- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown).toContain("Firecrawl");@@ -96,7 +100,7 @@ describe("Scrape tests", () => {const status = await scrapeStatus(response.metadata.scrapeId!);expect(JSON.stringify(status)).toBe(JSON.stringify(response));- }, 70000);+ }, scrapeTimeout);// describe("Ad blocking (f-e dependant)", () => {// it.concurrent("blocks ads by default", async () => {@@ -117,36 +121,36 @@ describe("Scrape tests", () => {// }, 30000);// });- describe("Index", () => {+ describe.only("Index", () => {it.concurrent("caches properly", async () => {const id = crypto.randomUUID();const url = "https://firecrawl.dev/?testId=" + id;const response1 = await scrape({url,- maxAge: 120000,+ maxAge: scrapeTimeout * 3,storeInCache: false,- timeout: 60000,+ timeout: scrapeTimeout,});expect(response1.metadata.cacheState).toBe("miss");- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,- maxAge: 120000,- timeout: 60000,+ maxAge: scrapeTimeout * 3,+ timeout: scrapeTimeout,});expect(response2.metadata.cacheState).toBe("miss");- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response3 = await scrape({url,- maxAge: 120000,- timeout: 60000,+ maxAge: scrapeTimeout * 3,+ timeout: scrapeTimeout,});expect(response3.metadata.cacheState).toBe("hit");@@ -155,11 +159,11 @@ describe("Scrape tests", () => {const response4 = await scrape({url,maxAge: 1,- timeout: 60000,+ timeout: scrapeTimeout,});expect(response4.metadata.cacheState).toBe("miss");- }, 150000 + 2 * 17000);+ }, scrapeTimeout * 4 + 2 * indexCooldown);it.concurrent("caches PDFs properly", async () => {const id = crypto.randomUUID();@@ -167,22 +171,22 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,});expect(response1.metadata.cacheState).toBe("miss");- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,});expect(response2.metadata.cacheState).toBe("hit");- }, 150000 + 2 * 17000);+ }, scrapeTimeout * 2 + 2 * indexCooldown);it.concurrent("respects screenshot", async () => {const id = crypto.randomUUID();@@ -191,18 +195,18 @@ describe("Scrape tests", () => {const response1 = await scrape({url,formats: ["screenshot"],- timeout: 60000,+ timeout: scrapeTimeout,});expect(response1.screenshot).toBeDefined();- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,formats: ["screenshot"],- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,});expect(response2.screenshot).toBe(response1.screenshot);@@ -210,13 +214,13 @@ describe("Scrape tests", () => {const response3 = await scrape({url,formats: ["screenshot@fullPage"],- timeout: 60000,- maxAge: 180000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 3,});expect(response3.screenshot).not.toBe(response1.screenshot);expect(response3.metadata.cacheState).toBe("miss");- }, 207000);+ }, scrapeTimeout * 3 + 2 * indexCooldown);it.concurrent("respects screenshot@fullPage", async () => {const id = crypto.randomUUID();@@ -225,18 +229,18 @@ describe("Scrape tests", () => {const response1 = await scrape({url,formats: ["screenshot@fullPage"],- timeout: 60000,+ timeout: scrapeTimeout,});expect(response1.screenshot).toBeDefined();- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,formats: ["screenshot@fullPage"],- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,});expect(response2.screenshot).toBe(response1.screenshot);@@ -244,13 +248,13 @@ describe("Scrape tests", () => {const response3 = await scrape({url,formats: ["screenshot"],- timeout: 60000,- maxAge: 180000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 3,});expect(response3.screenshot).not.toBe(response1.screenshot);expect(response3.metadata.cacheState).toBe("miss");- }, 207000);+ }, scrapeTimeout * 3 + 1 * indexCooldown);it.concurrent("respects changeTracking", async () => {const id = crypto.randomUUID();@@ -259,29 +263,29 @@ describe("Scrape tests", () => {await scrape({url,formats: ["markdown", "changeTracking"],- timeout: 60000,+ timeout: scrapeTimeout,});const response1 = await scrape({url,formats: ["markdown", "changeTracking"],- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,});expect(response1.metadata.cacheState).not.toBeDefined();- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,formats: ["markdown"],- timeout: 60000,- maxAge: 180000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 3 + indexCooldown,});expect(response2.metadata.cacheState).toBe("hit");- }, 147000);+ }, scrapeTimeout * 3 + 2 * indexCooldown);it.concurrent("respects headers", async () => {const id = crypto.randomUUID();@@ -292,19 +296,19 @@ describe("Scrape tests", () => {headers: {"X-Test": "test",},- timeout: 60000,+ timeout: scrapeTimeout,});- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response = await scrape({url,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2 + indexCooldown,});expect(response.metadata.cacheState).toBe("miss");- }, 147000);+ }, scrapeTimeout * 2 + 1 * indexCooldown);it.concurrent("respects mobile", async () => {const id = crypto.randomUUID();@@ -312,31 +316,31 @@ describe("Scrape tests", () => {await scrape({url,- timeout: 60000,+ timeout: scrapeTimeout,});- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response1 = await scrape({url,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,mobile: true,});expect(response1.metadata.cacheState).toBe("miss");- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,- timeout: 60000,- maxAge: 180000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 3,mobile: true,});expect(response2.metadata.cacheState).toBe("hit");- }, 224000);+ }, scrapeTimeout * 3 + 2 * indexCooldown);it.concurrent("respects actions", async () => {const id = crypto.randomUUID();@@ -344,8 +348,8 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- timeout: 60000,- maxAge: 60000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout,actions: [{"type": "wait","milliseconds": 1000,@@ -354,48 +358,48 @@ describe("Scrape tests", () => {expect(response1.metadata.cacheState).not.toBeDefined();- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,});expect(response2.metadata.cacheState).toBe("miss");- }, 147000);+ }, scrapeTimeout * 2 + 1 * indexCooldown);- it.concurrent("respects location", async () => {+ it.only("respects location", async () => {const id = crypto.randomUUID();const url = "https://firecrawl.dev/?testId=" + id;await scrape({url,- timeout: 60000,+ timeout: scrapeTimeout,});- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response1 = await scrape({url,- location: { country: "BR" },- maxAge: 120000,- timeout: 60000,+ location: { country: "DE" },+ maxAge: scrapeTimeout * 2,+ timeout: scrapeTimeout,});expect(response1.metadata.cacheState).toBe("miss");- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,- location: { country: "BR" },- timeout: 60000,- maxAge: 180000,+ location: { country: "DE" },+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 3,});expect(response2.metadata.cacheState).toBe("hit");- }, 224000);+ }, scrapeTimeout * 3 + 2 * indexCooldown);it.concurrent("respects blockAds", async () => {const id = crypto.randomUUID();@@ -404,16 +408,16 @@ describe("Scrape tests", () => {await scrape({url,blockAds: true,- timeout: 60000,+ timeout: scrapeTimeout,});- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response0 = await scrape({url,blockAds: true,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2 + indexCooldown,});expect(response0.metadata.cacheState).toBe("hit");@@ -421,23 +425,23 @@ describe("Scrape tests", () => {const response1 = await scrape({url,blockAds: false,- timeout: 60000,- maxAge: 180000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 3 + indexCooldown,});expect(response1.metadata.cacheState).toBe("miss");- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,blockAds: false,- timeout: 60000,- maxAge: 240000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 4 + 2 * indexCooldown,});expect(response2.metadata.cacheState).toBe("hit");- }, 284000);+ }, scrapeTimeout * 4 + 2 * indexCooldown);it.concurrent("respects proxy: stealth", async () => {const id = crypto.randomUUID();@@ -446,19 +450,19 @@ describe("Scrape tests", () => {const response1 = await scrape({url,proxy: "stealth",- timeout: 60000,- maxAge: 60000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout,});expect(response1.metadata.proxyUsed).toBe("stealth");expect(response1.metadata.cacheState).not.toBeDefined();- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response2 = await scrape({url,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2 + indexCooldown,});expect(response2.metadata.cacheState).toBe("hit");@@ -466,12 +470,12 @@ describe("Scrape tests", () => {const response3 = await scrape({url,proxy: "stealth",- timeout: 60000,- maxAge: 180000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 3 + indexCooldown,});expect(response3.metadata.cacheState).not.toBeDefined();- }, 207000);+ }, scrapeTimeout * 3 + indexCooldown);it.concurrent("works properly on pages returning 200", async () => {const id = crypto.randomUUID();@@ -479,19 +483,19 @@ describe("Scrape tests", () => {await scrape({url,- timeout: 60000,+ timeout: scrapeTimeout,});- await new Promise(resolve => setTimeout(resolve, 17000));+ await new Promise(resolve => setTimeout(resolve, indexCooldown));const response = await scrape({url,- timeout: 60000,- maxAge: 120000,+ timeout: scrapeTimeout,+ maxAge: scrapeTimeout * 2,});expect(response.metadata.cacheState).toBe("hit");- }, 147000);+ }, scrapeTimeout * 2 + 1 * indexCooldown);});describe("Change Tracking format", () => {@@ -499,12 +503,12 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://example.com",formats: ["markdown", "changeTracking"],- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();- }, 60000);+ }, scrapeTimeout);it.concurrent("includes git diff when requested", async () => {const response = await scrape({@@ -513,7 +517,7 @@ describe("Scrape tests", () => {changeTrackingOptions: {modes: ["git-diff"]},- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.changeTracking).toBeDefined();@@ -525,7 +529,7 @@ describe("Scrape tests", () => {expect(response.changeTracking?.diff?.json).toBeDefined();expect(response.changeTracking?.diff?.json.files).toBeInstanceOf(Array);}- }, 60000);+ }, scrapeTimeout);it.concurrent("includes structured output when requested", async () => {const response = await scrape({@@ -535,7 +539,7 @@ describe("Scrape tests", () => {modes: ["json"],prompt: "Summarize the changes between the previous and current content",},- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.changeTracking).toBeDefined();@@ -544,7 +548,7 @@ describe("Scrape tests", () => {if (response.changeTracking?.changeStatus === "changed") {expect(response.changeTracking?.json).toBeDefined();}- }, 60000);+ }, scrapeTimeout);it.concurrent("supports schema-based extraction for change tracking", async () => {const response = await scrape({@@ -569,7 +573,7 @@ describe("Scrape tests", () => {}}},- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.changeTracking).toBeDefined();@@ -586,7 +590,7 @@ describe("Scrape tests", () => {expect(response.changeTracking?.json.features).toHaveProperty("new");}}- }, 60000);+ }, scrapeTimeout);it.concurrent("supports both git-diff and structured modes together", async () => {const response = await scrape({@@ -602,7 +606,7 @@ describe("Scrape tests", () => {}}},- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.changeTracking).toBeDefined();@@ -617,7 +621,7 @@ describe("Scrape tests", () => {expect(response.changeTracking?.json).toHaveProperty("summary");expect(response.changeTracking?.json).toHaveProperty("changes");}- }, 60000);+ }, scrapeTimeout);it.concurrent("supports tags properly", async () => {const uuid1 = crypto.randomUUID();@@ -627,14 +631,14 @@ describe("Scrape tests", () => {url: "https://firecrawl.dev/",formats: ["markdown", "changeTracking"],changeTrackingOptions: { tag: uuid1 },- timeout: 60000,+ timeout: scrapeTimeout,});const response2 = await scrape({url: "https://firecrawl.dev/",formats: ["markdown", "changeTracking"],changeTrackingOptions: { tag: uuid2 },- timeout: 60000,+ timeout: scrapeTimeout,});expect(response1.changeTracking?.previousScrapeAt).toBeNull();@@ -646,31 +650,31 @@ describe("Scrape tests", () => {url: "https://firecrawl.dev/",formats: ["markdown", "changeTracking"],changeTrackingOptions: { tag: uuid1 },- timeout: 60000,+ timeout: scrapeTimeout,});expect(response3.changeTracking?.previousScrapeAt).not.toBeNull();expect(response3.changeTracking?.changeStatus).not.toBe("new");- }, 180000);+ }, scrapeTimeout * 3);});describe("Location API (f-e dependant)", () => {it.concurrent("works without specifying an explicit location", async () => {await scrape({url: "https://iplocation.com",- timeout: 60000,+ timeout: scrapeTimeout,});- }, 70000);+ }, scrapeTimeout);it.concurrent("works with country US", async () => {const response = await scrape({url: "https://iplocation.com",location: { country: "US" },- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown).toContain("| Country | United States |");- }, 70000);+ }, scrapeTimeout);});describe("Screenshot (f-e/sb dependant)", () => {@@ -678,66 +682,66 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",formats: ["screenshot"],- timeout: 60000,+ timeout: scrapeTimeout,});expect(typeof response.screenshot).toBe("string");- }, 70000);+ }, scrapeTimeout);it.concurrent("screenshot@fullPage format works", async () => {const response = await scrape({url: "http://firecrawl.dev",formats: ["screenshot@fullPage"],- timeout: 60000,+ timeout: scrapeTimeout,});expect(typeof response.screenshot).toBe("string");- }, 70000);+ }, scrapeTimeout);});describe("Proxy API (f-e dependant)", () => {it.concurrent("undefined works", async () => {await scrape({url: "http://firecrawl.dev",- timeout: 60000,+ timeout: scrapeTimeout,});- }, 70000);+ }, scrapeTimeout);it.concurrent("basic works", async () => {await scrape({url: "http://firecrawl.dev",proxy: "basic",- timeout: 60000,+ timeout: scrapeTimeout,});- }, 70000);+ }, scrapeTimeout);it.concurrent("stealth works", async () => {await scrape({url: "http://firecrawl.dev",proxy: "stealth",- timeout: 120000,+ timeout: scrapeTimeout * 2,});- }, 140000);+ }, scrapeTimeout * 2);it.concurrent("auto works properly on non-stealth site", async () => {const res = await scrape({url: "http://firecrawl.dev",proxy: "auto",- timeout: 120000,+ timeout: scrapeTimeout * 2,});expect(res.metadata.proxyUsed).toBe("basic");- }, 140000);+ }, scrapeTimeout * 2);it.concurrent("auto works properly on 'stealth' site (faked for reliabile testing)", async () => {const res = await scrape({url: "https://httpstat.us/403",proxy: "auto",- timeout: 120000,+ timeout: scrapeTimeout * 2,});expect(res.metadata.proxyUsed).toBe("stealth");- }, 140000);+ }, scrapeTimeout * 2);});describe("PDF (f-e dependant)", () => {@@ -753,39 +757,39 @@ describe("Scrape tests", () => {it.concurrent("blocks long PDFs with insufficient timeout", async () => {const response = await scrapeWithFailure({url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",- timeout: 30000,+ timeout: scrapeTimeout,});expect(response.error).toContain("Insufficient time to process PDF");- }, 30000);+ }, scrapeTimeout);it.concurrent("scrapes long PDFs with sufficient timeout", async () => {const response = await scrape({url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",- timeout: 300000,+ timeout: scrapeTimeout * 5,});// text on the last pageexpect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");- }, 310000);+ }, scrapeTimeout * 5);it.concurrent("scrapes Google Docs links as PDFs", async () => {const response = await scrape({url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",- timeout: 300000,+ timeout: scrapeTimeout * 5,});expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");- }, 310000);+ }, scrapeTimeout * 5);it.concurrent("scrapes Google Slides links as PDFs", async () => {const response = await scrape({url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view",- timeout: 300000,+ timeout: scrapeTimeout * 5,});expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities.");- }, 310000);+ }, scrapeTimeout * 5);});}@@ -813,7 +817,7 @@ describe("Scrape tests", () => {required: ["company_mission", "supports_sso", "is_open_source"],},},- timeout: 60000,+ timeout: scrapeTimeout,});expect(response).toHaveProperty("json");@@ -825,26 +829,26 @@ describe("Scrape tests", () => {expect(response.json).toHaveProperty("is_open_source");expect(response.json.is_open_source).toBe(true);expect(typeof response.json.is_open_source).toBe("boolean");- }, 60000);+ }, scrapeTimeout);});}it.concurrent("sourceURL stays unnormalized", async () => {const response = await scrape({url: "https://firecrawl.dev/?pagewanted=all&et_blog",- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");- }, 60000);+ }, scrapeTimeout);it.concurrent("application/json content type is markdownified properly", async () => {const response = await scrape({url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["markdown"],- timeout: 60000,+ timeout: scrapeTimeout,});expect(response.markdown).toContain("```json");- }, 60000);+ }, scrapeTimeout);});diff --git a/apps/api/src/__tests__/snips/webhook.test.ts b/apps/api/src/__tests__/snips/webhook.test.tsindex 86a5db258..e602d511c 100644--- a/apps/api/src/__tests__/snips/webhook.test.ts+++ b/apps/api/src/__tests__/snips/webhook.test.ts@@ -36,6 +36,9 @@ describe("Webhook tests", () => {url: `http://localhost:${WEBHOOK_PORT_CRAWL}/webhook`,},});++ // wait to settle the webhook calls+ await new Promise(resolve => setTimeout(resolve, 1000));server.close();@@ -92,6 +95,9 @@ describe("Webhook tests", () => {},});+ // wait to settle the webhook calls+ await new Promise(resolve => setTimeout(resolve, 1000));+server.close();let hasStarted = false;