Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api): CI (#1692)

GergΕ‘ MΓ³ricz9174e0c8a0c6
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index 8903458a3..6690926bc 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -41,7 +41,7 @@ describe("Crawl tests", () => {
}
}, 120000);
- it.concurrent("delay parameter works", async () => {
+ it.only("delay parameter works", async () => {
await crawl({
url: "https://firecrawl.dev",
limit: 3,
diff --git a/apps/api/src/__tests__/snips/extract.test.ts b/apps/api/src/__tests__/snips/extract.test.ts
index acba310d0..b305d3be6 100644
--- a/apps/api/src/__tests__/snips/extract.test.ts
+++ b/apps/api/src/__tests__/snips/extract.test.ts
@@ -20,6 +20,9 @@ describe("Extract tests", () => {
"is_open_source"
]
},
+ scrapeOptions: {
+ timeout: 75000,
+ },
origin: "api-sdk",
});
@@ -28,7 +31,7 @@ describe("Extract tests", () => {
expect(res.data).toHaveProperty("is_open_source");
expect(typeof res.data.is_open_source).toBe("boolean");
expect(res.data.is_open_source).toBe(true);
- }, 60000);
+ }, 90000);
it.concurrent("works with unsupported JSON schema parameters", async () => {
const res = await extract({
@@ -45,12 +48,15 @@ describe("Extract tests", () => {
"company_name"
]
},
+ scrapeOptions: {
+ timeout: 75000,
+ },
origin: "api-sdk",
});
expect(res.data).toHaveProperty("company_name");
expect(typeof res.data.company_name).toBe("string")
- }, 60000);
+ }, 90000);
} else {
it.concurrent("dummy test", () => {
expect(true).toBe(true);
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index c5479d199..a21d9f312 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -1,7 +1,7 @@
import { configDotenv } from "dotenv";
configDotenv();
-import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse } from "../../controllers/v1/types";
+import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse, CrawlErrorsResponse } from "../../controllers/v1/types";
import request from "supertest";
// =========================================
@@ -147,6 +147,18 @@ export async function asyncCrawlWaitForFinish(id: string, identity = defaultIden
return x.body;
}
+export async function crawlErrors(id: string, identity = defaultIdentity): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {
+ const res = await request(TEST_URL)
+ .get("/v1/crawl/" + id + "/errors")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .send();
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).not.toBe(false);
+
+ return res.body;
+}
+
export async function crawl(body: CrawlRequestInput, identity = defaultIdentity): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {
const cs = await crawlStart(body, identity);
expectCrawlStartToSucceed(cs);
@@ -159,6 +171,11 @@ export async function crawl(body: CrawlRequestInput, identity = defaultIdentity)
expect(typeof x.body.status).toBe("string");
} while (x.body.status === "scraping");
+ const errors = await crawlErrors(cs.body.id, identity);
+ if (errors.errors.length > 0) {
+ console.warn("Crawl ", cs.body.id, " had errors:", errors.errors);
+ }
+
expectCrawlToSucceed(x);
return x.body;
}
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 27ea6b2a7..f8066a655 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -1,6 +1,10 @@
import { scrape, scrapeStatus, scrapeWithFailure } from "./lib";
import crypto from "crypto";
+// Due to the limited resources of the CI runner, we need to set a longer timeout for the many many scrape tests
+const scrapeTimeout = 75000;
+const indexCooldown = 30000;
+
describe("Scrape tests", () => {
it.concurrent("mocking works properly", async () => {
// depends on falsified mock mocking-works-properly
@@ -10,51 +14,51 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "http://firecrawl.dev",
useMock: "mocking-works-properly",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown).toBe(
"this is fake data coming from the mocking system!",
);
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("works", async () => {
const response = await scrape({
url: "http://firecrawl.dev",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown).toContain("Firecrawl");
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("handles non-UTF-8 encodings", async () => {
const response = await scrape({
url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown).toContain("ぐ け げ こ ご さ ざ し じ す ず せ ぜ そ ぞ た");
- }, 60000);
+ }, scrapeTimeout);
if (process.env.TEST_SUITE_SELF_HOSTED && process.env.PROXY_SERVER) {
it.concurrent("self-hosted proxy works", async () => {
const response = await scrape({
url: "https://icanhazip.com",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);
- }, 70000);
+ }, scrapeTimeout);
it.concurrent("self-hosted proxy works on playwright", async () => {
const response = await scrape({
url: "https://icanhazip.com",
waitFor: 100,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);
- }, 70000);
+ }, scrapeTimeout);
}
if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.PLAYWRIGHT_MICROSERVICE_URL) {
@@ -62,11 +66,11 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "http://firecrawl.dev",
waitFor: 2000,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown).toContain("Firecrawl");
- }, 60000);
+ }, scrapeTimeout);
}
describe("JSON scrape support", () => {
@@ -74,19 +78,19 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://jsonplaceholder.typicode.com/todos/1",
formats: ["rawHtml"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
const obj = JSON.parse(response.rawHtml!);
expect(obj.id).toBe(1);
- }, 70000);
+ }, scrapeTimeout);
});
if (!process.env.TEST_SUITE_SELF_HOSTED) {
it.concurrent("scrape status works", async () => {
const response = await scrape({
url: "http://firecrawl.dev",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown).toContain("Firecrawl");
@@ -96,7 +100,7 @@ describe("Scrape tests", () => {
const status = await scrapeStatus(response.metadata.scrapeId!);
expect(JSON.stringify(status)).toBe(JSON.stringify(response));
- }, 70000);
+ }, scrapeTimeout);
// describe("Ad blocking (f-e dependant)", () => {
// it.concurrent("blocks ads by default", async () => {
@@ -117,36 +121,36 @@ describe("Scrape tests", () => {
// }, 30000);
// });
- describe("Index", () => {
+ describe.only("Index", () => {
it.concurrent("caches properly", async () => {
const id = crypto.randomUUID();
const url = "https://firecrawl.dev/?testId=" + id;
const response1 = await scrape({
url,
- maxAge: 120000,
+ maxAge: scrapeTimeout * 3,
storeInCache: false,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response1.metadata.cacheState).toBe("miss");
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
- maxAge: 120000,
- timeout: 60000,
+ maxAge: scrapeTimeout * 3,
+ timeout: scrapeTimeout,
});
expect(response2.metadata.cacheState).toBe("miss");
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response3 = await scrape({
url,
- maxAge: 120000,
- timeout: 60000,
+ maxAge: scrapeTimeout * 3,
+ timeout: scrapeTimeout,
});
expect(response3.metadata.cacheState).toBe("hit");
@@ -155,11 +159,11 @@ describe("Scrape tests", () => {
const response4 = await scrape({
url,
maxAge: 1,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response4.metadata.cacheState).toBe("miss");
- }, 150000 + 2 * 17000);
+ }, scrapeTimeout * 4 + 2 * indexCooldown);
it.concurrent("caches PDFs properly", async () => {
const id = crypto.randomUUID();
@@ -167,22 +171,22 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
});
expect(response1.metadata.cacheState).toBe("miss");
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
});
expect(response2.metadata.cacheState).toBe("hit");
- }, 150000 + 2 * 17000);
+ }, scrapeTimeout * 2 + 2 * indexCooldown);
it.concurrent("respects screenshot", async () => {
const id = crypto.randomUUID();
@@ -191,18 +195,18 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
formats: ["screenshot"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response1.screenshot).toBeDefined();
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
formats: ["screenshot"],
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
});
expect(response2.screenshot).toBe(response1.screenshot);
@@ -210,13 +214,13 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url,
formats: ["screenshot@fullPage"],
- timeout: 60000,
- maxAge: 180000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 3,
});
expect(response3.screenshot).not.toBe(response1.screenshot);
expect(response3.metadata.cacheState).toBe("miss");
- }, 207000);
+ }, scrapeTimeout * 3 + 2 * indexCooldown);
it.concurrent("respects screenshot@fullPage", async () => {
const id = crypto.randomUUID();
@@ -225,18 +229,18 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
formats: ["screenshot@fullPage"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response1.screenshot).toBeDefined();
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
formats: ["screenshot@fullPage"],
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
});
expect(response2.screenshot).toBe(response1.screenshot);
@@ -244,13 +248,13 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url,
formats: ["screenshot"],
- timeout: 60000,
- maxAge: 180000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 3,
});
expect(response3.screenshot).not.toBe(response1.screenshot);
expect(response3.metadata.cacheState).toBe("miss");
- }, 207000);
+ }, scrapeTimeout * 3 + 1 * indexCooldown);
it.concurrent("respects changeTracking", async () => {
const id = crypto.randomUUID();
@@ -259,29 +263,29 @@ describe("Scrape tests", () => {
await scrape({
url,
formats: ["markdown", "changeTracking"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
const response1 = await scrape({
url,
formats: ["markdown", "changeTracking"],
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
});
expect(response1.metadata.cacheState).not.toBeDefined();
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
formats: ["markdown"],
- timeout: 60000,
- maxAge: 180000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 3 + indexCooldown,
});
expect(response2.metadata.cacheState).toBe("hit");
- }, 147000);
+ }, scrapeTimeout * 3 + 2 * indexCooldown);
it.concurrent("respects headers", async () => {
const id = crypto.randomUUID();
@@ -292,19 +296,19 @@ describe("Scrape tests", () => {
headers: {
"X-Test": "test",
},
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response = await scrape({
url,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2 + indexCooldown,
});
expect(response.metadata.cacheState).toBe("miss");
- }, 147000);
+ }, scrapeTimeout * 2 + 1 * indexCooldown);
it.concurrent("respects mobile", async () => {
const id = crypto.randomUUID();
@@ -312,31 +316,31 @@ describe("Scrape tests", () => {
await scrape({
url,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response1 = await scrape({
url,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
mobile: true,
});
expect(response1.metadata.cacheState).toBe("miss");
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
- timeout: 60000,
- maxAge: 180000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 3,
mobile: true,
});
expect(response2.metadata.cacheState).toBe("hit");
- }, 224000);
+ }, scrapeTimeout * 3 + 2 * indexCooldown);
it.concurrent("respects actions", async () => {
const id = crypto.randomUUID();
@@ -344,8 +348,8 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
- timeout: 60000,
- maxAge: 60000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout,
actions: [{
"type": "wait",
"milliseconds": 1000,
@@ -354,48 +358,48 @@ describe("Scrape tests", () => {
expect(response1.metadata.cacheState).not.toBeDefined();
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
});
expect(response2.metadata.cacheState).toBe("miss");
- }, 147000);
+ }, scrapeTimeout * 2 + 1 * indexCooldown);
- it.concurrent("respects location", async () => {
+ it.only("respects location", async () => {
const id = crypto.randomUUID();
const url = "https://firecrawl.dev/?testId=" + id;
await scrape({
url,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response1 = await scrape({
url,
- location: { country: "BR" },
- maxAge: 120000,
- timeout: 60000,
+ location: { country: "DE" },
+ maxAge: scrapeTimeout * 2,
+ timeout: scrapeTimeout,
});
expect(response1.metadata.cacheState).toBe("miss");
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
- location: { country: "BR" },
- timeout: 60000,
- maxAge: 180000,
+ location: { country: "DE" },
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 3,
});
expect(response2.metadata.cacheState).toBe("hit");
- }, 224000);
+ }, scrapeTimeout * 3 + 2 * indexCooldown);
it.concurrent("respects blockAds", async () => {
const id = crypto.randomUUID();
@@ -404,16 +408,16 @@ describe("Scrape tests", () => {
await scrape({
url,
blockAds: true,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response0 = await scrape({
url,
blockAds: true,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2 + indexCooldown,
});
expect(response0.metadata.cacheState).toBe("hit");
@@ -421,23 +425,23 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
blockAds: false,
- timeout: 60000,
- maxAge: 180000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 3 + indexCooldown,
});
expect(response1.metadata.cacheState).toBe("miss");
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
blockAds: false,
- timeout: 60000,
- maxAge: 240000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 4 + 2 * indexCooldown,
});
expect(response2.metadata.cacheState).toBe("hit");
- }, 284000);
+ }, scrapeTimeout * 4 + 2 * indexCooldown);
it.concurrent("respects proxy: stealth", async () => {
const id = crypto.randomUUID();
@@ -446,19 +450,19 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
proxy: "stealth",
- timeout: 60000,
- maxAge: 60000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout,
});
expect(response1.metadata.proxyUsed).toBe("stealth");
expect(response1.metadata.cacheState).not.toBeDefined();
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response2 = await scrape({
url,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2 + indexCooldown,
});
expect(response2.metadata.cacheState).toBe("hit");
@@ -466,12 +470,12 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url,
proxy: "stealth",
- timeout: 60000,
- maxAge: 180000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 3 + indexCooldown,
});
expect(response3.metadata.cacheState).not.toBeDefined();
- }, 207000);
+ }, scrapeTimeout * 3 + indexCooldown);
it.concurrent("works properly on pages returning 200", async () => {
const id = crypto.randomUUID();
@@ -479,19 +483,19 @@ describe("Scrape tests", () => {
await scrape({
url,
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- await new Promise(resolve => setTimeout(resolve, 17000));
+ await new Promise(resolve => setTimeout(resolve, indexCooldown));
const response = await scrape({
url,
- timeout: 60000,
- maxAge: 120000,
+ timeout: scrapeTimeout,
+ maxAge: scrapeTimeout * 2,
});
expect(response.metadata.cacheState).toBe("hit");
- }, 147000);
+ }, scrapeTimeout * 2 + 1 * indexCooldown);
});
describe("Change Tracking format", () => {
@@ -499,12 +503,12 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://example.com",
formats: ["markdown", "changeTracking"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.changeTracking).toBeDefined();
expect(response.changeTracking?.previousScrapeAt).not.toBeNull();
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("includes git diff when requested", async () => {
const response = await scrape({
@@ -513,7 +517,7 @@ describe("Scrape tests", () => {
changeTrackingOptions: {
modes: ["git-diff"]
},
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.changeTracking).toBeDefined();
@@ -525,7 +529,7 @@ describe("Scrape tests", () => {
expect(response.changeTracking?.diff?.json).toBeDefined();
expect(response.changeTracking?.diff?.json.files).toBeInstanceOf(Array);
}
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("includes structured output when requested", async () => {
const response = await scrape({
@@ -535,7 +539,7 @@ describe("Scrape tests", () => {
modes: ["json"],
prompt: "Summarize the changes between the previous and current content",
},
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.changeTracking).toBeDefined();
@@ -544,7 +548,7 @@ describe("Scrape tests", () => {
if (response.changeTracking?.changeStatus === "changed") {
expect(response.changeTracking?.json).toBeDefined();
}
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("supports schema-based extraction for change tracking", async () => {
const response = await scrape({
@@ -569,7 +573,7 @@ describe("Scrape tests", () => {
}
}
},
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.changeTracking).toBeDefined();
@@ -586,7 +590,7 @@ describe("Scrape tests", () => {
expect(response.changeTracking?.json.features).toHaveProperty("new");
}
}
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("supports both git-diff and structured modes together", async () => {
const response = await scrape({
@@ -602,7 +606,7 @@ describe("Scrape tests", () => {
}
}
},
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.changeTracking).toBeDefined();
@@ -617,7 +621,7 @@ describe("Scrape tests", () => {
expect(response.changeTracking?.json).toHaveProperty("summary");
expect(response.changeTracking?.json).toHaveProperty("changes");
}
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("supports tags properly", async () => {
const uuid1 = crypto.randomUUID();
@@ -627,14 +631,14 @@ describe("Scrape tests", () => {
url: "https://firecrawl.dev/",
formats: ["markdown", "changeTracking"],
changeTrackingOptions: { tag: uuid1 },
- timeout: 60000,
+ timeout: scrapeTimeout,
});
const response2 = await scrape({
url: "https://firecrawl.dev/",
formats: ["markdown", "changeTracking"],
changeTrackingOptions: { tag: uuid2 },
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response1.changeTracking?.previousScrapeAt).toBeNull();
@@ -646,31 +650,31 @@ describe("Scrape tests", () => {
url: "https://firecrawl.dev/",
formats: ["markdown", "changeTracking"],
changeTrackingOptions: { tag: uuid1 },
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response3.changeTracking?.previousScrapeAt).not.toBeNull();
expect(response3.changeTracking?.changeStatus).not.toBe("new");
- }, 180000);
+ }, scrapeTimeout * 3);
});
describe("Location API (f-e dependant)", () => {
it.concurrent("works without specifying an explicit location", async () => {
await scrape({
url: "https://iplocation.com",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- }, 70000);
+ }, scrapeTimeout);
it.concurrent("works with country US", async () => {
const response = await scrape({
url: "https://iplocation.com",
location: { country: "US" },
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown).toContain("| Country | United States |");
- }, 70000);
+ }, scrapeTimeout);
});
describe("Screenshot (f-e/sb dependant)", () => {
@@ -678,66 +682,66 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "http://firecrawl.dev",
formats: ["screenshot"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(typeof response.screenshot).toBe("string");
- }, 70000);
+ }, scrapeTimeout);
it.concurrent("screenshot@fullPage format works", async () => {
const response = await scrape({
url: "http://firecrawl.dev",
formats: ["screenshot@fullPage"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(typeof response.screenshot).toBe("string");
- }, 70000);
+ }, scrapeTimeout);
});
describe("Proxy API (f-e dependant)", () => {
it.concurrent("undefined works", async () => {
await scrape({
url: "http://firecrawl.dev",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- }, 70000);
+ }, scrapeTimeout);
it.concurrent("basic works", async () => {
await scrape({
url: "http://firecrawl.dev",
proxy: "basic",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
- }, 70000);
+ }, scrapeTimeout);
it.concurrent("stealth works", async () => {
await scrape({
url: "http://firecrawl.dev",
proxy: "stealth",
- timeout: 120000,
+ timeout: scrapeTimeout * 2,
});
- }, 140000);
+ }, scrapeTimeout * 2);
it.concurrent("auto works properly on non-stealth site", async () => {
const res = await scrape({
url: "http://firecrawl.dev",
proxy: "auto",
- timeout: 120000,
+ timeout: scrapeTimeout * 2,
});
expect(res.metadata.proxyUsed).toBe("basic");
- }, 140000);
+ }, scrapeTimeout * 2);
it.concurrent("auto works properly on 'stealth' site (faked for reliabile testing)", async () => {
const res = await scrape({
url: "https://httpstat.us/403",
proxy: "auto",
- timeout: 120000,
+ timeout: scrapeTimeout * 2,
});
expect(res.metadata.proxyUsed).toBe("stealth");
- }, 140000);
+ }, scrapeTimeout * 2);
});
describe("PDF (f-e dependant)", () => {
@@ -753,39 +757,39 @@ describe("Scrape tests", () => {
it.concurrent("blocks long PDFs with insufficient timeout", async () => {
const response = await scrapeWithFailure({
url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.error).toContain("Insufficient time to process PDF");
- }, 30000);
+ }, scrapeTimeout);
it.concurrent("scrapes long PDFs with sufficient timeout", async () => {
const response = await scrape({
url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",
- timeout: 300000,
+ timeout: scrapeTimeout * 5,
});
// text on the last page
expect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");
- }, 310000);
+ }, scrapeTimeout * 5);
it.concurrent("scrapes Google Docs links as PDFs", async () => {
const response = await scrape({
url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",
- timeout: 300000,
+ timeout: scrapeTimeout * 5,
});
expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");
- }, 310000);
+ }, scrapeTimeout * 5);
it.concurrent("scrapes Google Slides links as PDFs", async () => {
const response = await scrape({
url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view",
- timeout: 300000,
+ timeout: scrapeTimeout * 5,
});
expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities.");
- }, 310000);
+ }, scrapeTimeout * 5);
});
}
@@ -813,7 +817,7 @@ describe("Scrape tests", () => {
required: ["company_mission", "supports_sso", "is_open_source"],
},
},
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response).toHaveProperty("json");
@@ -825,26 +829,26 @@ describe("Scrape tests", () => {
expect(response.json).toHaveProperty("is_open_source");
expect(response.json.is_open_source).toBe(true);
expect(typeof response.json.is_open_source).toBe("boolean");
- }, 60000);
+ }, scrapeTimeout);
});
}
it.concurrent("sourceURL stays unnormalized", async () => {
const response = await scrape({
url: "https://firecrawl.dev/?pagewanted=all&et_blog",
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");
- }, 60000);
+ }, scrapeTimeout);
it.concurrent("application/json content type is markdownified properly", async () => {
const response = await scrape({
url: "https://jsonplaceholder.typicode.com/todos/1",
formats: ["markdown"],
- timeout: 60000,
+ timeout: scrapeTimeout,
});
expect(response.markdown).toContain("```json");
- }, 60000);
+ }, scrapeTimeout);
});
diff --git a/apps/api/src/__tests__/snips/webhook.test.ts b/apps/api/src/__tests__/snips/webhook.test.ts
index 86a5db258..e602d511c 100644
--- a/apps/api/src/__tests__/snips/webhook.test.ts
+++ b/apps/api/src/__tests__/snips/webhook.test.ts
@@ -36,6 +36,9 @@ describe("Webhook tests", () => {
url: `http://localhost:${WEBHOOK_PORT_CRAWL}/webhook`,
},
});
+
+ // wait to settle the webhook calls
+ await new Promise(resolve => setTimeout(resolve, 1000));
server.close();
@@ -92,6 +95,9 @@ describe("Webhook tests", () => {
},
});
+ // wait to settle the webhook calls
+ await new Promise(resolve => setTimeout(resolve, 1000));
+
server.close();
let hasStarted = false;