Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/ci): idmux (#1707)
GergΕ MΓ³riczae94edd43ec8
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex ca30d1de6..074d1ac79 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -42,6 +42,7 @@ env:VERTEX_CREDENTIALS: ${{ secrets.VERTEX_CREDENTIALS }}USE_GO_MARKDOWN_PARSER: trueSENTRY_ENVIRONMENT: dev+ IDMUX_URL: ${{ secrets.IDMUX_URL }}jobs:test:diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.tsindex 9a3f3cba6..3e03b92dc 100644--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts@@ -1,10 +1,20 @@-import { batchScrape, scrapeTimeout } from "./lib";+import { batchScrape, scrapeTimeout, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "batch-scrape",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000);describe("Batch scrape tests", () => {it.concurrent("works", async () => {const response = await batchScrape({urls: ["http://firecrawl.dev"]- });+ }, identity);expect(response.data[0]).toHaveProperty("markdown");expect(response.data[0].markdown).toContain("Firecrawl");@@ -34,7 +44,7 @@ describe("Batch scrape tests", () => {required: ["company_mission", "supports_sso", "is_open_source"],},},- });+ }, identity);expect(response.data[0]).toHaveProperty("json");expect(response.data[0].json).toHaveProperty("company_mission");@@ -52,7 +62,7 @@ describe("Batch scrape tests", () => {it.concurrent("sourceURL stays unnormalized", async () => {const response = await batchScrape({urls: ["https://firecrawl.dev/?pagewanted=all&et_blog"],- });+ }, identity);expect(response.data[0].metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");}, scrapeTimeout);diff --git a/apps/api/src/__tests__/snips/billing.test.ts b/apps/api/src/__tests__/snips/billing.test.tsindex 2b54572e0..d1e004438 100644--- a/apps/api/src/__tests__/snips/billing.test.ts+++ b/apps/api/src/__tests__/snips/billing.test.ts@@ -1,35 +1,33 @@-import { batchScrape, crawl, creditUsage, extract, map, scrape, search, tokenUsage } from "./lib";+import { batchScrape, crawl, creditUsage, extract, idmux, map, scrape, search, tokenUsage } from "./lib";const sleep = (ms: number) => new Promise(x => setTimeout(() => x(true), ms));const sleepForBatchBilling = () => sleep(40000);-beforeAll(async () => {- // Wait for previous test runs to stop billing processing- if (!process.env.TEST_SUITE_SELF_HOSTED) {- await sleep(40000);- }-}, 50000);-describe("Billing tests", () => {if (process.env.TEST_SUITE_SELF_HOSTED) {it("dummy", () => {expect(true).toBe(true);});} else {- it("bills scrape correctly", async () => {- const rc1 = (await creditUsage()).remaining_credits;+ it.concurrent("bills scrape correctly", async () => {+ const identity = await idmux({+ name: "billing/bills scrape correctly",+ credits: 100,+ });++ const rc1 = (await creditUsage(identity)).remaining_credits;// Run all scrape operations in parallel with Promise.allconst [scrape1, scrape2, scrape3] = await Promise.all([// scrape 1: regular fc.dev scrape (1 credit)scrape({url: "https://firecrawl.dev"- }),+ }, identity),// scrape 1.1: regular fc.dev scrape (1 credit)scrape({url: "https://firecrawl.dev"- }),+ }, identity),// scrape 2: fc.dev with json (5 credits)scrape({@@ -44,7 +42,7 @@ describe("Billing tests", () => {required: ["is_open_source"],},},- })+ }, identity)]);expect(scrape1.metadata.creditsUsed).toBe(1);@@ -55,13 +53,18 @@ describe("Billing tests", () => {await sleepForBatchBilling();- const rc2 = (await creditUsage()).remaining_credits;+ const rc2 = (await creditUsage(identity)).remaining_credits;expect(rc1 - rc2).toBe(7);}, 120000);- it("bills batch scrape correctly", async () => {- const rc1 = (await creditUsage()).remaining_credits;+ it.concurrent("bills batch scrape correctly", async () => {+ const identity = await idmux({+ name: "billing/bills batch scrape correctly",+ credits: 100,+ });++ const rc1 = (await creditUsage(identity)).remaining_credits;// Run both scrape operations in parallel with Promise.allconst [scrape1, scrape2] = await Promise.all([@@ -72,7 +75,7 @@ describe("Billing tests", () => {"https://mendable.ai","https://thisdomaindoesnotexistandwillfail.fcr",],- }),+ }, identity),// scrape 2: batch scrape with json (10 credits)batchScrape({@@ -91,7 +94,7 @@ describe("Billing tests", () => {required: ["four_word_summary"],},},- })+ }, identity)]);expect(scrape1.data[0].metadata.creditsUsed).toBe(1);@@ -104,13 +107,18 @@ describe("Billing tests", () => {await sleepForBatchBilling();- const rc2 = (await creditUsage()).remaining_credits;+ const rc2 = (await creditUsage(identity)).remaining_credits;expect(rc1 - rc2).toBe(12);}, 600000);- it("bills crawl correctly", async () => {- const rc1 = (await creditUsage()).remaining_credits;+ it.concurrent("bills crawl correctly", async () => {+ const identity = await idmux({+ name: "billing/bills crawl correctly",+ credits: 200,+ });++ const rc1 = (await creditUsage(identity)).remaining_credits;// Run both crawl operations in parallel with Promise.allconst [crawl1, crawl2] = await Promise.all([@@ -118,7 +126,7 @@ describe("Billing tests", () => {crawl({url: "https://firecrawl.dev",limit: 10,- }),+ }, identity),// crawl 2: fc.dev crawl with json (5y credits)crawl({@@ -136,7 +144,7 @@ describe("Billing tests", () => {},},limit: 10,- })+ }, identity)]);expect(crawl1.success).toBe(true);@@ -146,54 +154,74 @@ describe("Billing tests", () => {await sleepForBatchBilling();- const rc2 = (await creditUsage()).remaining_credits;+ const rc2 = (await creditUsage(identity)).remaining_credits;if (crawl1.success && crawl2.success) {expect(rc1 - rc2).toBe(crawl1.completed + crawl2.completed * 5);}}, 600000);- it("bills map correctly", async () => {- const rc1 = (await creditUsage()).remaining_credits;- await map({ url: "https://firecrawl.dev" });+ it.concurrent("bills map correctly", async () => {+ const identity = await idmux({+ name: "billing/bills map correctly",+ credits: 100,+ });++ const rc1 = (await creditUsage(identity)).remaining_credits;+ await map({ url: "https://firecrawl.dev" }, identity);await sleepForBatchBilling();- const rc2 = (await creditUsage()).remaining_credits;+ const rc2 = (await creditUsage(identity)).remaining_credits;expect(rc1 - rc2).toBe(1);}, 60000);- it("bills search correctly", async () => {- const rc1 = (await creditUsage()).remaining_credits;+ it.concurrent("bills search correctly", async () => {+ const identity = await idmux({+ name: "billing/bills search correctly",+ credits: 100,+ });++ const rc1 = (await creditUsage(identity)).remaining_credits;const results = await search({query: "firecrawl"- });+ }, identity);await sleepForBatchBilling();- const rc2 = (await creditUsage()).remaining_credits;+ const rc2 = (await creditUsage(identity)).remaining_credits;expect(rc1 - rc2).toBe(results.length);}, 60000);- it("bills search with scrape correctly", async () => {- const rc1 = (await creditUsage()).remaining_credits;+ it.concurrent("bills search with scrape correctly", async () => {+ const identity = await idmux({+ name: "billing/bills search with scrape correctly",+ credits: 100,+ });++ const rc1 = (await creditUsage(identity)).remaining_credits;const results = await search({query: "firecrawl",scrapeOptions: {formats: ["markdown"],},- });+ }, identity);await sleepForBatchBilling();- const rc2 = (await creditUsage()).remaining_credits;+ const rc2 = (await creditUsage(identity)).remaining_credits;expect(rc1 - rc2).toBe(results.length);}, 600000);- it("bills extract correctly", async () => {- const rc1 = (await tokenUsage()).remaining_tokens;+ it.concurrent("bills extract correctly", async () => {+ const identity = await idmux({+ name: "billing/bills extract correctly",+ tokens: 1000,+ });++ const rc1 = (await tokenUsage(identity)).remaining_tokens;const extractResult = await extract({urls: ["https://firecrawl.dev"],@@ -209,13 +237,13 @@ describe("Billing tests", () => {]},origin: "api-sdk",- });+ }, identity);expect(extractResult.tokensUsed).toBe(305);await sleepForBatchBilling();- const rc2 = (await tokenUsage()).remaining_tokens;+ const rc2 = (await tokenUsage(identity)).remaining_tokens;expect(rc1 - rc2).toBe(305);}, 300000);diff --git a/apps/api/src/__tests__/snips/concurrency.test.ts b/apps/api/src/__tests__/snips/concurrency.test.tsindex 3128145aa..f38f291cd 100644--- a/apps/api/src/__tests__/snips/concurrency.test.ts+++ b/apps/api/src/__tests__/snips/concurrency.test.ts@@ -1,59 +1,62 @@-import { batchScrapeWithConcurrencyTracking, concurrencyCheck, crawlWithConcurrencyTracking, defaultIdentity, Identity } from "./lib";--const concurrencyIdentity: Identity = {- apiKey: process.env.TEST_API_KEY_CONCURRENCY ?? process.env.TEST_API_KEY!,-}+import { batchScrapeWithConcurrencyTracking, crawlWithConcurrencyTracking, idmux } from "./lib";if (!process.env.TEST_SUITE_SELF_HOSTED) {- let accountConcurrencyLimit = 2;-- beforeAll(async () => {- const { maxConcurrency } = await concurrencyCheck(concurrencyIdentity);- accountConcurrencyLimit = maxConcurrency;+ const accountConcurrencyLimit = 20;- console.log("Account concurrency limit:", accountConcurrencyLimit);-- if (accountConcurrencyLimit > 20) {- console.warn("Your account's concurrency limit (" + accountConcurrencyLimit + ") is likely too high, which will cause these tests to fail. Please set up TEST_API_KEY_CONCURRENCY with an API key that has a lower concurrency limit.");- }- }, 10000);-describe("Concurrency queue and limit", () => {- it("crawl utilizes full concurrency limit and doesn't go over", async () => {+ it.concurrent("crawl utilizes full concurrency limit and doesn't go over", async () => {+ const identity = await idmux({+ name: "concurrency/crawl utilizes full concurrency limit and doesn't go over",+ concurrency: accountConcurrencyLimit,+ credits: 100,+ });+const limit = accountConcurrencyLimit * 2;const { crawl, concurrencies } = await crawlWithConcurrencyTracking({url: "https://firecrawl.dev",limit,- }, concurrencyIdentity);+ }, identity);expect(Math.max(...concurrencies)).toBe(accountConcurrencyLimit);expect(crawl.completed).toBe(limit);}, 600000);- it("crawl handles maxConcurrency properly", async () => {+ it.concurrent("crawl handles maxConcurrency properly", async () => {+ const identity = await idmux({+ name: "concurrency/crawl handles maxConcurrency properly",+ concurrency: accountConcurrencyLimit,+ credits: 100,+ });+const { crawl, concurrencies } = await crawlWithConcurrencyTracking({url: "https://firecrawl.dev",limit: 15,maxConcurrency: 5,- }, concurrencyIdentity);+ }, identity);expect(Math.max(...concurrencies)).toBe(5);expect(crawl.completed).toBe(15);}, 600000);- it("crawl maxConcurrency stacks properly", async () => {+ it.concurrent("crawl maxConcurrency stacks properly", async () => {+ const identity = await idmux({+ name: "concurrency/crawl maxConcurrency stacks properly",+ concurrency: accountConcurrencyLimit,+ credits: 100,+ });+const [{ crawl: crawl1, concurrencies: concurrencies1 }, { crawl: crawl2, concurrencies: concurrencies2 }] = await Promise.all([crawlWithConcurrencyTracking({url: "https://firecrawl.dev",limit: 15,maxConcurrency: 5,- }, concurrencyIdentity),+ }, identity),crawlWithConcurrencyTracking({url: "https://firecrawl.dev",limit: 15,maxConcurrency: 5,- }, concurrencyIdentity),+ }, identity),]);expect(Math.max(...concurrencies1, ...concurrencies2)).toBe(10);@@ -61,37 +64,55 @@ if (!process.env.TEST_SUITE_SELF_HOSTED) {expect(crawl2.completed).toBe(15);}, 1200000);- it("batch scrape utilizes full concurrency limit and doesn't go over", async () => {+ it.concurrent("batch scrape utilizes full concurrency limit and doesn't go over", async () => {+ const identity = await idmux({+ name: "concurrency/batch scrape utilizes full concurrency limit and doesn't go over",+ concurrency: accountConcurrencyLimit,+ credits: 100,+ });+const limit = accountConcurrencyLimit * 2;const { batchScrape, concurrencies } = await batchScrapeWithConcurrencyTracking({urls: Array(limit).fill(0).map(_ => `https://firecrawl.dev`),- }, concurrencyIdentity);+ }, identity);expect(Math.max(...concurrencies)).toBe(accountConcurrencyLimit);expect(batchScrape.completed).toBe(limit);}, 600000);- it("batch scrape handles maxConcurrency properly", async () => {+ it.concurrent("batch scrape handles maxConcurrency properly", async () => {+ const identity = await idmux({+ name: "concurrency/batch scrape handles maxConcurrency properly",+ concurrency: accountConcurrencyLimit,+ credits: 100,+ });+const { batchScrape, concurrencies } = await batchScrapeWithConcurrencyTracking({urls: Array(15).fill(0).map(_ => `https://firecrawl.dev`),maxConcurrency: 5,- }, concurrencyIdentity);+ }, identity);expect(Math.max(...concurrencies)).toBe(5);expect(batchScrape.completed).toBe(15);}, 600000);- it("batch scrape maxConcurrency stacks properly", async () => {+ it.concurrent("batch scrape maxConcurrency stacks properly", async () => {+ const identity = await idmux({+ name: "concurrency/batch scrape maxConcurrency stacks properly",+ concurrency: accountConcurrencyLimit,+ credits: 100,+ });+const [{ batchScrape: batchScrape1, concurrencies: concurrencies1 }, { batchScrape: batchScrape2, concurrencies: concurrencies2 }] = await Promise.all([batchScrapeWithConcurrencyTracking({urls: Array(15).fill(0).map(_ => `https://firecrawl.dev`),maxConcurrency: 5,- }, concurrencyIdentity),+ }, identity),batchScrapeWithConcurrencyTracking({urls: Array(15).fill(0).map(_ => `https://firecrawl.dev`),maxConcurrency: 5,- }, concurrencyIdentity),+ }, identity),]);expect(Math.max(...concurrencies1, ...concurrencies2)).toBe(10);diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex 5f57776b9..3a565fc52 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -1,12 +1,22 @@-import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing, scrapeTimeout } from "./lib";+import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing, Identity, idmux, scrapeTimeout } from "./lib";import { describe, it, expect } from "@jest/globals";+let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "crawl",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000);+describe("Crawl tests", () => {it.concurrent("works", async () => {await crawl({url: "https://firecrawl.dev",limit: 10,- });+ }, identity);}, 10 * scrapeTimeout);it.concurrent("filters URLs properly", async () => {@@ -14,7 +24,7 @@ describe("Crawl tests", () => {url: "https://firecrawl.dev/pricing",includePaths: ["^/pricing$"],limit: 10,- });+ }, identity);expect(res.success).toBe(true);if (res.success) {@@ -32,7 +42,7 @@ describe("Crawl tests", () => {includePaths: ["^https://(www\\.)?firecrawl\\.dev/pricing$"],regexOnFullURL: true,limit: 10,- });+ }, identity);expect(res.success).toBe(true);if (res.success) {@@ -46,22 +56,22 @@ describe("Crawl tests", () => {url: "https://firecrawl.dev",limit: 3,delay: 5,- });+ }, identity);}, 3 * scrapeTimeout + 3 * 5000);it.concurrent("ongoing crawls endpoint works", async () => {const res = await asyncCrawl({url: "https://firecrawl.dev",limit: 3,- });+ }, identity);- const ongoing = await crawlOngoing();+ const ongoing = await crawlOngoing(identity);expect(ongoing.crawls.find(x => x.id === res.id)).toBeDefined();- await asyncCrawlWaitForFinish(res.id);+ await asyncCrawlWaitForFinish(res.id, identity);- const ongoing2 = await crawlOngoing();+ const ongoing2 = await crawlOngoing(identity);expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();}, 3 * scrapeTimeout);@@ -106,7 +116,7 @@ describe("Crawl tests", () => {url: "https://firecrawl.dev",crawlEntireDomain: true,limit: 5,- });+ }, identity);expect(res.success).toBe(true);if (res.success) {@@ -120,7 +130,7 @@ describe("Crawl tests", () => {allowBackwardLinks: false,crawlEntireDomain: true,limit: 5,- });+ }, identity);expect(res.success).toBe(true);if (res.success) {@@ -133,7 +143,7 @@ describe("Crawl tests", () => {url: "https://firecrawl.dev",allowBackwardLinks: true,limit: 5,- });+ }, identity);expect(res.success).toBe(true);if (res.success) {diff --git a/apps/api/src/__tests__/snips/extract.test.ts b/apps/api/src/__tests__/snips/extract.test.tsindex b305d3be6..c3e5abeed 100644--- a/apps/api/src/__tests__/snips/extract.test.ts+++ b/apps/api/src/__tests__/snips/extract.test.ts@@ -1,4 +1,14 @@-import { extract } from "./lib";+import { extract, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "extract",+ concurrency: 100,+ tokens: 1000000,+ });+}, 10000);describe("Extract tests", () => {if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {@@ -24,7 +34,7 @@ describe("Extract tests", () => {timeout: 75000,},origin: "api-sdk",- });+ }, identity);expect(res.data).toHaveProperty("company_mission");expect(typeof res.data.company_mission).toBe("string")@@ -52,7 +62,7 @@ describe("Extract tests", () => {timeout: 75000,},origin: "api-sdk",- });+ }, identity);expect(res.data).toHaveProperty("company_name");expect(typeof res.data.company_name).toBe("string")diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 6d0e28fd2..d39db3fc4 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -10,23 +10,70 @@ import request from "supertest";const TEST_URL = "http://127.0.0.1:3002";-export type Identity = {- apiKey: string;+// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests+export const scrapeTimeout = 90000;+export const indexCooldown = 30000;++// =========================================+// idmux+// =========================================++export type IdmuxRequest = {+ name: string,++ concurrency?: number,+ credits?: number,+ tokens?: number,+ flags?: any,}-export const defaultIdentity: Identity = {- apiKey: process.env.TEST_API_KEY!,-};+export async function idmux(req: IdmuxRequest): Promise<Identity> {+ if (!process.env.IDMUX_URL) {+ if (!process.env.TEST_SUITE_SELF_HOSTED) {+ console.warn("IDMUX_URL is not set, using test API key and team ID");+ }+ return {+ apiKey: process.env.TEST_API_KEY!,+ teamId: process.env.TEST_TEAM_ID!,+ }+ }-// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests-export const scrapeTimeout = 75000;-export const indexCooldown = 30000;+ let runNumber = parseInt(process.env.GITHUB_RUN_NUMBER!);+ if (isNaN(runNumber) || runNumber === null || runNumber === undefined) {+ runNumber = 0;+ }++ const res = await fetch(process.env.IDMUX_URL + "/", {+ method: "POST",+ body: JSON.stringify({+ refName: process.env.GITHUB_REF_NAME!,+ runNumber,+ concurrency: req.concurrency ?? 100,+ ...req,+ }),+ headers: {+ "Content-Type": "application/json",+ },+ });++ if (!res.ok) {+ console.error(await res.text());+ }++ expect(res.ok).toBe(true);+ return await res.json();+}++export type Identity = {+ apiKey: string;+ teamId: string;+}// =========================================// Scrape API// =========================================-async function scrapeRaw(body: ScrapeRequestInput, identity = defaultIdentity) {+async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {return await request(TEST_URL).post("/v1/scrape").set("Authorization", `Bearer ${identity.apiKey}`)@@ -46,7 +93,7 @@ function expectScrapeToFail(response: Awaited<ReturnType<typeof scrapeRaw>>) {expect(typeof response.body.error).toBe("string");}-export async function scrape(body: ScrapeRequestInput, identity = defaultIdentity): Promise<Document> {+export async function scrape(body: ScrapeRequestInput, identity: Identity): Promise<Document> {const raw = await scrapeRaw(body, identity);expectScrapeToSucceed(raw);if (body.proxy === "stealth") {@@ -57,7 +104,7 @@ export async function scrape(body: ScrapeRequestInput, identity = defaultIdentitreturn raw.body.data;}-export async function scrapeWithFailure(body: ScrapeRequestInput, identity = defaultIdentity): Promise<{+export async function scrapeWithFailure(body: ScrapeRequestInput, identity: Identity): Promise<{success: false;error: string;}> {@@ -66,14 +113,14 @@ export async function scrapeWithFailure(body: ScrapeRequestInput, identity = defreturn raw.body;}-export async function scrapeStatusRaw(jobId: string, identity = defaultIdentity) {+export async function scrapeStatusRaw(jobId: string, identity: Identity) {return await request(TEST_URL).get("/v1/scrape/" + encodeURIComponent(jobId)).set("Authorization", `Bearer ${identity.apiKey}`).send();}-export async function scrapeStatus(jobId: string, identity = defaultIdentity): Promise<Document> {+export async function scrapeStatus(jobId: string, identity: Identity): Promise<Document> {const raw = await scrapeStatusRaw(jobId, identity);expect(raw.statusCode).toBe(200);expect(raw.body.success).toBe(true);@@ -87,7 +134,7 @@ export async function scrapeStatus(jobId: string, identity = defaultIdentity): P// Crawl API// =========================================-async function crawlStart(body: CrawlRequestInput, identity = defaultIdentity) {+async function crawlStart(body: CrawlRequestInput, identity: Identity) {return await request(TEST_URL).post("/v1/crawl").set("Authorization", `Bearer ${identity.apiKey}`)@@ -95,21 +142,21 @@ async function crawlStart(body: CrawlRequestInput, identity = defaultIdentity) {.send(body);}-async function crawlStatus(id: string, identity = defaultIdentity) {+async function crawlStatus(id: string, identity: Identity) {return await request(TEST_URL).get("/v1/crawl/" + encodeURIComponent(id)).set("Authorization", `Bearer ${identity.apiKey}`).send();}-async function crawlOngoingRaw(identity = defaultIdentity) {+async function crawlOngoingRaw(identity: Identity) {return await request(TEST_URL).get("/v1/crawl/ongoing").set("Authorization", `Bearer ${identity.apiKey}`).send();}-export async function crawlOngoing(identity = defaultIdentity): Promise<Exclude<OngoingCrawlsResponse, ErrorResponse>> {+export async function crawlOngoing(identity: Identity): Promise<Exclude<OngoingCrawlsResponse, ErrorResponse>> {const res = await crawlOngoingRaw(identity);expect(res.statusCode).toBe(200);expect(res.body.success).toBe(true);@@ -132,13 +179,13 @@ function expectCrawlToSucceed(response: Awaited<ReturnType<typeof crawlStatus>>)expect(response.body.data.length).toBeGreaterThan(0);}-export async function asyncCrawl(body: CrawlRequestInput, identity = defaultIdentity): Promise<Exclude<CrawlResponse, ErrorResponse>> {+export async function asyncCrawl(body: CrawlRequestInput, identity: Identity): Promise<Exclude<CrawlResponse, ErrorResponse>> {const cs = await crawlStart(body, identity);expectCrawlStartToSucceed(cs);return cs.body;}-export async function asyncCrawlWaitForFinish(id: string, identity = defaultIdentity): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {+export async function asyncCrawlWaitForFinish(id: string, identity: Identity): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {let x;do {@@ -151,7 +198,7 @@ export async function asyncCrawlWaitForFinish(id: string, identity = defaultIdenreturn x.body;}-export async function crawlErrors(id: string, identity = defaultIdentity): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {+export async function crawlErrors(id: string, identity: Identity): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {const res = await request(TEST_URL).get("/v1/crawl/" + id + "/errors").set("Authorization", `Bearer ${identity.apiKey}`)@@ -163,7 +210,7 @@ export async function crawlErrors(id: string, identity = defaultIdentity): Promireturn res.body;}-export async function crawl(body: CrawlRequestInput, identity = defaultIdentity): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {+export async function crawl(body: CrawlRequestInput, identity: Identity): Promise<Exclude<CrawlStatusResponse & { id: string }, ErrorResponse>> {const cs = await crawlStart(body, identity);expectCrawlStartToSucceed(cs);@@ -188,7 +235,7 @@ export async function crawl(body: CrawlRequestInput, identity = defaultIdentity)// Batch Scrape API// =========================================-async function batchScrapeStart(body: BatchScrapeRequestInput, identity = defaultIdentity) {+async function batchScrapeStart(body: BatchScrapeRequestInput, identity: Identity) {return await request(TEST_URL).post("/v1/batch/scrape").set("Authorization", `Bearer ${identity.apiKey}`)@@ -196,7 +243,7 @@ async function batchScrapeStart(body: BatchScrapeRequestInput, identity = defaul.send(body);}-async function batchScrapeStatus(id: string, identity = defaultIdentity) {+async function batchScrapeStatus(id: string, identity: Identity) {return await request(TEST_URL).get("/v1/batch/scrape/" + encodeURIComponent(id)).set("Authorization", `Bearer ${identity.apiKey}`)@@ -219,7 +266,7 @@ function expectBatchScrapeToSucceed(response: Awaited<ReturnType<typeof batchScrexpect(response.body.data.length).toBeGreaterThan(0);}-export async function batchScrape(body: BatchScrapeRequestInput, identity = defaultIdentity): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {+export async function batchScrape(body: BatchScrapeRequestInput, identity: Identity): Promise<Exclude<CrawlStatusResponse, ErrorResponse> & { id: string }> {const bss = await batchScrapeStart(body, identity);expectBatchScrapeStartToSucceed(bss);@@ -239,7 +286,7 @@ export async function batchScrape(body: BatchScrapeRequestInput, identity = defa// Map API// =========================================-export async function map(body: MapRequestInput, identity = defaultIdentity) {+export async function map(body: MapRequestInput, identity: Identity) {return await request(TEST_URL).post("/v1/map").set("Authorization", `Bearer ${identity.apiKey}`)@@ -258,7 +305,7 @@ export function expectMapToSucceed(response: Awaited<ReturnType<typeof map>>) {// Extract API// =========================================-async function extractStart(body: ExtractRequestInput, identity = defaultIdentity) {+async function extractStart(body: ExtractRequestInput, identity: Identity) {return await request(TEST_URL).post("/v1/extract").set("Authorization", `Bearer ${identity.apiKey}`)@@ -266,7 +313,7 @@ async function extractStart(body: ExtractRequestInput, identity = defaultIdentit.send(body);}-async function extractStatus(id: string, identity = defaultIdentity) {+async function extractStatus(id: string, identity: Identity) {return await request(TEST_URL).get("/v1/extract/" + encodeURIComponent(id)).set("Authorization", `Bearer ${identity.apiKey}`)@@ -288,7 +335,7 @@ function expectExtractToSucceed(response: Awaited<ReturnType<typeof extractStatuexpect(response.body).toHaveProperty("data");}-export async function extract(body: ExtractRequestInput, identity = defaultIdentity): Promise<ExtractResponse> {+export async function extract(body: ExtractRequestInput, identity: Identity): Promise<ExtractResponse> {const es = await extractStart(body, identity);expectExtractStartToSucceed(es);@@ -308,7 +355,7 @@ export async function extract(body: ExtractRequestInput, identity = defaultIdent// Search API// =========================================-async function searchRaw(body: SearchRequestInput, identity = defaultIdentity) {+async function searchRaw(body: SearchRequestInput, identity: Identity) {return await request(TEST_URL).post("/v1/search").set("Authorization", `Bearer ${identity.apiKey}`)@@ -324,7 +371,7 @@ function expectSearchToSucceed(response: Awaited<ReturnType<typeof searchRaw>>)expect(response.body.data.length).toBeGreaterThan(0);}-export async function search(body: SearchRequestInput, identity = defaultIdentity): Promise<Document[]> {+export async function search(body: SearchRequestInput, identity: Identity): Promise<Document[]> {const raw = await searchRaw(body, identity);expectSearchToSucceed(raw);return raw.body.data;@@ -334,7 +381,7 @@ export async function search(body: SearchRequestInput, identity = defaultIdentit// Billing API// =========================================-export async function creditUsage(identity = defaultIdentity): Promise<{ remaining_credits: number }> {+export async function creditUsage(identity: Identity): Promise<{ remaining_credits: number }> {const req = (await request(TEST_URL).get("/v1/team/credit-usage").set("Authorization", `Bearer ${identity.apiKey}`)@@ -347,7 +394,7 @@ export async function creditUsage(identity = defaultIdentity): Promise<{ remainireturn req.body.data;}-export async function tokenUsage(identity = defaultIdentity): Promise<{ remaining_tokens: number }> {+export async function tokenUsage(identity: Identity): Promise<{ remaining_tokens: number }> {return (await request(TEST_URL).get("/v1/team/token-usage").set("Authorization", `Bearer ${identity.apiKey}`)@@ -358,7 +405,7 @@ export async function tokenUsage(identity = defaultIdentity): Promise<{ remainin// Concurrency API// =========================================-export async function concurrencyCheck(identity = defaultIdentity): Promise<{ concurrency: number, maxConcurrency: number }> {+export async function concurrencyCheck(identity: Identity): Promise<{ concurrency: number, maxConcurrency: number }> {const x = (await request(TEST_URL).get("/v1/concurrency-check").set("Authorization", `Bearer ${identity.apiKey}`)@@ -369,7 +416,7 @@ export async function concurrencyCheck(identity = defaultIdentity): Promise<{ coreturn x.body;}-export async function crawlWithConcurrencyTracking(body: CrawlRequestInput, identity = defaultIdentity): Promise<{+export async function crawlWithConcurrencyTracking(body: CrawlRequestInput, identity: Identity): Promise<{crawl: Exclude<CrawlStatusResponse, ErrorResponse>;concurrencies: number[];}> {@@ -392,7 +439,7 @@ export async function crawlWithConcurrencyTracking(body: CrawlRequestInput, iden};}-export async function batchScrapeWithConcurrencyTracking(body: BatchScrapeRequestInput, identity = defaultIdentity): Promise<{+export async function batchScrapeWithConcurrencyTracking(body: BatchScrapeRequestInput, identity: Identity): Promise<{batchScrape: Exclude<CrawlStatusResponse, ErrorResponse>;concurrencies: number[];}> {@@ -428,7 +475,7 @@ async function deepResearchStart(body: {formats?: string[];topic?: string;jsonOptions?: any;-}, identity = defaultIdentity) {+}, identity: Identity) {return await request(TEST_URL).post("/v1/deep-research").set("Authorization", `Bearer ${identity.apiKey}`)@@ -436,7 +483,7 @@ async function deepResearchStart(body: {.send(body);}-async function deepResearchStatus(id: string, identity = defaultIdentity) {+async function deepResearchStatus(id: string, identity: Identity) {return await request(TEST_URL).get("/v1/deep-research/" + encodeURIComponent(id)).set("Authorization", `Bearer ${identity.apiKey}`)@@ -459,7 +506,7 @@ export async function deepResearch(body: {formats?: string[];topic?: string;jsonOptions?: any;-}, identity = defaultIdentity) {+}, identity: Identity) {const ds = await deepResearchStart(body, identity);expectDeepResearchStartToSucceed(ds);diff --git a/apps/api/src/__tests__/snips/map.test.ts b/apps/api/src/__tests__/snips/map.test.tsindex 8c9ffe055..304c9dd39 100644--- a/apps/api/src/__tests__/snips/map.test.ts+++ b/apps/api/src/__tests__/snips/map.test.ts@@ -1,19 +1,29 @@-import { expectMapToSucceed, map } from "./lib";+import { expectMapToSucceed, map, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "map",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000);describe("Map tests", () => {it.concurrent("basic map succeeds", async () => {const response = await map({url: "http://firecrawl.dev",- });+ }, identity);expectMapToSucceed(response);- }, 10000);+ }, 60000);it.concurrent("times out properly", async () => {const response = await map({url: "http://firecrawl.dev",timeout: 1- });+ }, identity);expect(response.statusCode).toBe(408);expect(response.body.success).toBe(false);@@ -25,7 +35,7 @@ describe("Map tests", () => {url: "https://www.hfea.gov.uk",sitemapOnly: true,useMock: "map-query-params",- });+ }, identity);expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 6c8c80d7e..0bcb34c1d 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -1,6 +1,25 @@-import { scrape, scrapeStatus, scrapeWithFailure, scrapeTimeout, indexCooldown } from "./lib";+import { scrape, scrapeStatus, scrapeWithFailure, scrapeTimeout, indexCooldown, idmux, Identity } from "./lib";import crypto from "crypto";+let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "scrape",+ concurrency: 100,+ credits: 1000000,+ });++ if (!process.env.TEST_SUITE_SELF_HOSTED) {+ // Needed for change tracking tests to work+ await scrape({+ url: "https://example.com",+ formats: ["markdown", "changeTracking"],+ timeout: scrapeTimeout,+ }, identity);+ }+}, 10000 + scrapeTimeout);+describe("Scrape tests", () => {it.concurrent("mocking works properly", async () => {// depends on falsified mock mocking-works-properly@@ -11,7 +30,7 @@ describe("Scrape tests", () => {url: "http://firecrawl.dev",useMock: "mocking-works-properly",timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown).toBe("this is fake data coming from the mocking system!",@@ -22,7 +41,7 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown).toContain("Firecrawl");}, scrapeTimeout);@@ -31,7 +50,7 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown).toContain("γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ");}, scrapeTimeout);@@ -41,7 +60,7 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://icanhazip.com",timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);}, scrapeTimeout);@@ -51,7 +70,7 @@ describe("Scrape tests", () => {url: "https://icanhazip.com",waitFor: 100,timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);}, scrapeTimeout);@@ -63,7 +82,7 @@ describe("Scrape tests", () => {url: "http://firecrawl.dev",waitFor: 2000,timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown).toContain("Firecrawl");}, scrapeTimeout);@@ -75,7 +94,7 @@ describe("Scrape tests", () => {url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["rawHtml"],timeout: scrapeTimeout,- });+ }, identity);const obj = JSON.parse(response.rawHtml!);expect(obj.id).toBe(1);@@ -87,35 +106,34 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown).toContain("Firecrawl");// Give time to propagate to read replicaawait new Promise(resolve => setTimeout(resolve, 1000));- const status = await scrapeStatus(response.metadata.scrapeId!);+ const status = await scrapeStatus(response.metadata.scrapeId!, identity);expect(JSON.stringify(status)).toBe(JSON.stringify(response));}, scrapeTimeout);- // describe("Ad blocking (f-e dependant)", () => {- // it.concurrent("blocks ads by default", async () => {- // const response = await scrape({- // url: "https://www.allrecipes.com/recipe/18185/yum/",- // });-- // expect(response.markdown).not.toContain(".g.doubleclick.net/");- // }, 30000);-- // it.concurrent("doesn't block ads if explicitly disabled", async () => {- // const response = await scrape({- // url: "https://www.allrecipes.com/recipe/18185/yum/",- // blockAds: false,- // });-- // expect(response.markdown).toMatch(/(\.g\.doubleclick\.net|amazon-adsystem\.com)\//);- // }, 30000);- // });+ describe("Ad blocking (f-e dependant)", () => {+ it.concurrent("blocking ads works", async () => {+ await scrape({+ url: "https://firecrawl.dev",+ blockAds: true,+ timeout: scrapeTimeout,+ }, identity);+ }, scrapeTimeout);++ it.concurrent("doesn't block ads if explicitly disabled", async () => {+ await scrape({+ url: "https://firecrawl.dev",+ blockAds: false,+ timeout: scrapeTimeout,+ }, identity);+ }, scrapeTimeout);+ });describe("Index", () => {it.concurrent("caches properly", async () => {@@ -127,7 +145,7 @@ describe("Scrape tests", () => {maxAge: scrapeTimeout * 3,storeInCache: false,timeout: scrapeTimeout,- });+ }, identity);expect(response1.metadata.cacheState).toBe("miss");@@ -137,7 +155,7 @@ describe("Scrape tests", () => {url,maxAge: scrapeTimeout * 3,timeout: scrapeTimeout,- });+ }, identity);expect(response2.metadata.cacheState).toBe("miss");@@ -147,7 +165,7 @@ describe("Scrape tests", () => {url,maxAge: scrapeTimeout * 3,timeout: scrapeTimeout,- });+ }, identity);expect(response3.metadata.cacheState).toBe("hit");expect(response3.metadata.cachedAt).toBeDefined();@@ -156,7 +174,7 @@ describe("Scrape tests", () => {url,maxAge: 1,timeout: scrapeTimeout,- });+ }, identity);expect(response4.metadata.cacheState).toBe("miss");}, scrapeTimeout * 4 + 2 * indexCooldown);@@ -169,7 +187,7 @@ describe("Scrape tests", () => {url,timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,- });+ }, identity);expect(response1.metadata.cacheState).toBe("miss");@@ -179,7 +197,7 @@ describe("Scrape tests", () => {url,timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,- });+ }, identity);expect(response2.metadata.cacheState).toBe("hit");}, scrapeTimeout * 2 + 2 * indexCooldown);@@ -192,7 +210,7 @@ describe("Scrape tests", () => {url,formats: ["screenshot"],timeout: scrapeTimeout,- });+ }, identity);expect(response1.screenshot).toBeDefined();@@ -203,7 +221,7 @@ describe("Scrape tests", () => {formats: ["screenshot"],timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,- });+ }, identity);expect(response2.screenshot).toBe(response1.screenshot);@@ -212,7 +230,7 @@ describe("Scrape tests", () => {formats: ["screenshot@fullPage"],timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,- });+ }, identity);expect(response3.screenshot).not.toBe(response1.screenshot);expect(response3.metadata.cacheState).toBe("miss");@@ -226,7 +244,7 @@ describe("Scrape tests", () => {url,formats: ["screenshot@fullPage"],timeout: scrapeTimeout,- });+ }, identity);expect(response1.screenshot).toBeDefined();@@ -237,7 +255,7 @@ describe("Scrape tests", () => {formats: ["screenshot@fullPage"],timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,- });+ }, identity);expect(response2.screenshot).toBe(response1.screenshot);@@ -246,7 +264,7 @@ describe("Scrape tests", () => {formats: ["screenshot"],timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,- });+ }, identity);expect(response3.screenshot).not.toBe(response1.screenshot);expect(response3.metadata.cacheState).toBe("miss");@@ -260,14 +278,14 @@ describe("Scrape tests", () => {url,formats: ["markdown", "changeTracking"],timeout: scrapeTimeout,- });+ }, identity);const response1 = await scrape({url,formats: ["markdown", "changeTracking"],timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,- });+ }, identity);expect(response1.metadata.cacheState).not.toBeDefined();@@ -278,7 +296,7 @@ describe("Scrape tests", () => {formats: ["markdown"],timeout: scrapeTimeout,maxAge: scrapeTimeout * 3 + indexCooldown,- });+ }, identity);expect(response2.metadata.cacheState).toBe("hit");}, scrapeTimeout * 3 + 2 * indexCooldown);@@ -293,7 +311,7 @@ describe("Scrape tests", () => {"X-Test": "test",},timeout: scrapeTimeout,- });+ }, identity);await new Promise(resolve => setTimeout(resolve, indexCooldown));@@ -301,7 +319,7 @@ describe("Scrape tests", () => {url,timeout: scrapeTimeout,maxAge: scrapeTimeout * 2 + indexCooldown,- });+ }, identity);expect(response.metadata.cacheState).toBe("miss");}, scrapeTimeout * 2 + 1 * indexCooldown);@@ -313,7 +331,7 @@ describe("Scrape tests", () => {await scrape({url,timeout: scrapeTimeout,- });+ }, identity);await new Promise(resolve => setTimeout(resolve, indexCooldown));@@ -322,7 +340,7 @@ describe("Scrape tests", () => {timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,mobile: true,- });+ }, identity);expect(response1.metadata.cacheState).toBe("miss");@@ -333,7 +351,7 @@ describe("Scrape tests", () => {timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,mobile: true,- });+ }, identity);expect(response2.metadata.cacheState).toBe("hit");}, scrapeTimeout * 3 + 2 * indexCooldown);@@ -350,7 +368,7 @@ describe("Scrape tests", () => {"type": "wait","milliseconds": 1000,}]- });+ }, identity);expect(response1.metadata.cacheState).not.toBeDefined();@@ -360,7 +378,7 @@ describe("Scrape tests", () => {url,timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,- });+ }, identity);expect(response2.metadata.cacheState).toBe("miss");}, scrapeTimeout * 2 + 1 * indexCooldown);@@ -372,7 +390,7 @@ describe("Scrape tests", () => {await scrape({url,timeout: scrapeTimeout,- });+ }, identity);await new Promise(resolve => setTimeout(resolve, indexCooldown));@@ -381,7 +399,7 @@ describe("Scrape tests", () => {location: { country: "DE" },maxAge: scrapeTimeout * 2,timeout: scrapeTimeout,- });+ }, identity);expect(response1.metadata.cacheState).toBe("miss");@@ -392,7 +410,7 @@ describe("Scrape tests", () => {location: { country: "DE" },timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,- });+ }, identity);expect(response2.metadata.cacheState).toBe("hit");}, scrapeTimeout * 3 + 2 * indexCooldown);@@ -405,7 +423,7 @@ describe("Scrape tests", () => {url,blockAds: true,timeout: scrapeTimeout,- });+ }, identity);await new Promise(resolve => setTimeout(resolve, indexCooldown));@@ -414,7 +432,7 @@ describe("Scrape tests", () => {blockAds: true,timeout: scrapeTimeout,maxAge: scrapeTimeout * 2 + indexCooldown,- });+ }, identity);expect(response0.metadata.cacheState).toBe("hit");@@ -423,7 +441,7 @@ describe("Scrape tests", () => {blockAds: false,timeout: scrapeTimeout,maxAge: scrapeTimeout * 3 + indexCooldown,- });+ }, identity);expect(response1.metadata.cacheState).toBe("miss");@@ -434,7 +452,7 @@ describe("Scrape tests", () => {blockAds: false,timeout: scrapeTimeout,maxAge: scrapeTimeout * 4 + 2 * indexCooldown,- });+ }, identity);expect(response2.metadata.cacheState).toBe("hit");}, scrapeTimeout * 4 + 2 * indexCooldown);@@ -448,7 +466,7 @@ describe("Scrape tests", () => {proxy: "stealth",timeout: scrapeTimeout,maxAge: scrapeTimeout,- });+ }, identity);expect(response1.metadata.proxyUsed).toBe("stealth");expect(response1.metadata.cacheState).not.toBeDefined();@@ -459,7 +477,7 @@ describe("Scrape tests", () => {url,timeout: scrapeTimeout,maxAge: scrapeTimeout * 2 + indexCooldown,- });+ }, identity);expect(response2.metadata.cacheState).toBe("hit");@@ -468,19 +486,19 @@ describe("Scrape tests", () => {proxy: "stealth",timeout: scrapeTimeout,maxAge: scrapeTimeout * 3 + indexCooldown,- });+ }, identity);expect(response3.metadata.cacheState).not.toBeDefined();}, scrapeTimeout * 3 + indexCooldown);it.concurrent("works properly on pages returning 200", async () => {const id = crypto.randomUUID();- const url = "https://httpstat.us/200?testId=" + id;+ const url = "https://firecrawl.dev/?testId=" + id;await scrape({url,timeout: scrapeTimeout,- });+ }, identity);await new Promise(resolve => setTimeout(resolve, indexCooldown));@@ -488,7 +506,7 @@ describe("Scrape tests", () => {url,timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,- });+ }, identity);expect(response.metadata.cacheState).toBe("hit");}, scrapeTimeout * 2 + 1 * indexCooldown);@@ -500,7 +518,7 @@ describe("Scrape tests", () => {url: "https://example.com",formats: ["markdown", "changeTracking"],timeout: scrapeTimeout,- });+ }, identity);expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();@@ -514,7 +532,7 @@ describe("Scrape tests", () => {modes: ["git-diff"]},timeout: scrapeTimeout,- });+ }, identity);expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();@@ -536,7 +554,7 @@ describe("Scrape tests", () => {prompt: "Summarize the changes between the previous and current content",},timeout: scrapeTimeout,- });+ }, identity);expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();@@ -570,7 +588,7 @@ describe("Scrape tests", () => {}},timeout: scrapeTimeout,- });+ }, identity);expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();@@ -603,7 +621,7 @@ describe("Scrape tests", () => {}},timeout: scrapeTimeout,- });+ }, identity);expect(response.changeTracking).toBeDefined();expect(response.changeTracking?.previousScrapeAt).not.toBeNull();@@ -628,14 +646,14 @@ describe("Scrape tests", () => {formats: ["markdown", "changeTracking"],changeTrackingOptions: { tag: uuid1 },timeout: scrapeTimeout,- });+ }, identity);const response2 = await scrape({url: "https://firecrawl.dev/",formats: ["markdown", "changeTracking"],changeTrackingOptions: { tag: uuid2 },timeout: scrapeTimeout,- });+ }, identity);expect(response1.changeTracking?.previousScrapeAt).toBeNull();expect(response1.changeTracking?.changeStatus).toBe("new");@@ -647,7 +665,7 @@ describe("Scrape tests", () => {formats: ["markdown", "changeTracking"],changeTrackingOptions: { tag: uuid1 },timeout: scrapeTimeout,- });+ }, identity);expect(response3.changeTracking?.previousScrapeAt).not.toBeNull();expect(response3.changeTracking?.changeStatus).not.toBe("new");@@ -659,7 +677,7 @@ describe("Scrape tests", () => {await scrape({url: "https://iplocation.com",timeout: scrapeTimeout,- });+ }, identity);}, scrapeTimeout);it.concurrent("works with country US", async () => {@@ -667,7 +685,7 @@ describe("Scrape tests", () => {url: "https://iplocation.com",location: { country: "US" },timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown).toContain("| Country | United States |");}, scrapeTimeout);@@ -679,7 +697,7 @@ describe("Scrape tests", () => {url: "http://firecrawl.dev",formats: ["screenshot"],timeout: scrapeTimeout,- });+ }, identity);expect(typeof response.screenshot).toBe("string");}, scrapeTimeout);@@ -689,7 +707,7 @@ describe("Scrape tests", () => {url: "http://firecrawl.dev",formats: ["screenshot@fullPage"],timeout: scrapeTimeout,- });+ }, identity);expect(typeof response.screenshot).toBe("string");}, scrapeTimeout);@@ -701,7 +719,7 @@ describe("Scrape tests", () => {url: "https://firecrawl.dev",timeout: scrapeTimeout,actions: [{ type: "pdf" }],- });+ }, identity);expect(response.actions?.pdfs).toBeDefined();expect(response.actions?.pdfs?.length).toBe(1);@@ -715,7 +733,7 @@ describe("Scrape tests", () => {await scrape({url: "http://firecrawl.dev",timeout: scrapeTimeout,- });+ }, identity);}, scrapeTimeout);it.concurrent("basic works", async () => {@@ -723,7 +741,7 @@ describe("Scrape tests", () => {url: "http://firecrawl.dev",proxy: "basic",timeout: scrapeTimeout,- });+ }, identity);}, scrapeTimeout);it.concurrent("stealth works", async () => {@@ -731,7 +749,7 @@ describe("Scrape tests", () => {url: "http://firecrawl.dev",proxy: "stealth",timeout: scrapeTimeout * 2,- });+ }, identity);}, scrapeTimeout * 2);it.concurrent("auto works properly on non-stealth site", async () => {@@ -739,20 +757,21 @@ describe("Scrape tests", () => {url: "http://firecrawl.dev",proxy: "auto",timeout: scrapeTimeout * 2,- });+ }, identity);expect(res.metadata.proxyUsed).toBe("basic");}, scrapeTimeout * 2);- it.concurrent("auto works properly on 'stealth' site (faked for reliabile testing)", async () => {- const res = await scrape({- url: "https://httpstat.us/403",- proxy: "auto",- timeout: scrapeTimeout * 2,- });+ // TODO: flaky+ // it.concurrent("auto works properly on 'stealth' site (faked for reliabile testing)", async () => {+ // const res = await scrape({+ // url: "https://eo16f6718vph4un.m.pipedream.net", // always returns 403+ // proxy: "auto",+ // timeout: scrapeTimeout * 2,+ // }, identity);- expect(res.metadata.proxyUsed).toBe("stealth");- }, scrapeTimeout * 2);+ // expect(res.metadata.proxyUsed).toBe("stealth");+ // }, scrapeTimeout * 2);});describe("PDF (f-e dependant)", () => {@@ -769,7 +788,7 @@ describe("Scrape tests", () => {const response = await scrapeWithFailure({url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",timeout: scrapeTimeout,- });+ }, identity);expect(response.error).toContain("Insufficient time to process PDF");}, scrapeTimeout);@@ -778,7 +797,7 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",timeout: scrapeTimeout * 5,- });+ }, identity);// text on the last pageexpect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");@@ -788,7 +807,7 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",timeout: scrapeTimeout * 5,- });+ }, identity);expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");}, scrapeTimeout * 5);@@ -797,7 +816,7 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view",timeout: scrapeTimeout * 5,- });+ }, identity);expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities.");}, scrapeTimeout * 5);@@ -829,7 +848,7 @@ describe("Scrape tests", () => {},},timeout: scrapeTimeout,- });+ }, identity);expect(response).toHaveProperty("json");expect(response.json).toHaveProperty("company_mission");@@ -848,7 +867,7 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://firecrawl.dev/?pagewanted=all&et_blog",timeout: scrapeTimeout,- });+ }, identity);expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");}, scrapeTimeout);@@ -858,7 +877,7 @@ describe("Scrape tests", () => {url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["markdown"],timeout: scrapeTimeout,- });+ }, identity);expect(response.markdown).toContain("```json");}, scrapeTimeout);diff --git a/apps/api/src/__tests__/snips/search.test.ts b/apps/api/src/__tests__/snips/search.test.tsindex 51ce5bf15..1a081a52f 100644--- a/apps/api/src/__tests__/snips/search.test.ts+++ b/apps/api/src/__tests__/snips/search.test.ts@@ -1,10 +1,20 @@-import { search } from "./lib";+import { search, idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "search",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000);describe("Search tests", () => {it.concurrent("works", async () => {await search({query: "firecrawl"- });+ }, identity);}, 60000);it.concurrent("works with scrape", async () => {@@ -15,7 +25,7 @@ describe("Search tests", () => {formats: ["markdown"],},timeout: 120000,- });+ }, identity);for (const doc of res) {expect(doc.markdown).toBeDefined();diff --git a/apps/api/src/__tests__/snips/webhook.test.ts b/apps/api/src/__tests__/snips/webhook.test.tsindex e602d511c..10feb719e 100644--- a/apps/api/src/__tests__/snips/webhook.test.ts+++ b/apps/api/src/__tests__/snips/webhook.test.ts@@ -1,4 +1,4 @@-import { crawl, batchScrape } from "./lib";+import { crawl, batchScrape, idmux, Identity } from "./lib";import Express from "express";import bodyParser from "body-parser";import type { WebhookEventType } from "src/types";@@ -7,6 +7,16 @@ import type { Document } from "src/controllers/v1/types";const WEBHOOK_PORT_CRAWL = 3008;const WEBHOOK_PORT_BATCH_SCRAPE = 3009;+let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "webhook",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000);+describe("Webhook tests", () => {it.concurrent("webhook works properly for crawl", async () => {const app = Express();@@ -35,7 +45,7 @@ describe("Webhook tests", () => {webhook: {url: `http://localhost:${WEBHOOK_PORT_CRAWL}/webhook`,},- });+ }, identity);// wait to settle the webhook callsawait new Promise(resolve => setTimeout(resolve, 1000));@@ -93,7 +103,7 @@ describe("Webhook tests", () => {webhook: {url: `http://localhost:${WEBHOOK_PORT_BATCH_SCRAPE}/webhook`,},- });+ }, identity);// wait to settle the webhook callsawait new Promise(resolve => setTimeout(resolve, 1000));