Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(v2/timeout): initial new waterfalling system (ENG-2922) (#1950)

GergΕ‘ MΓ³ricz9c001a917a8f
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.ts
index 83e71366d..23c031df7 100644
--- a/apps/api/src/__tests__/snips/v2/map.test.ts
+++ b/apps/api/src/__tests__/snips/v2/map.test.ts
@@ -33,7 +33,7 @@ describe("Map tests", () => {
it.concurrent("handles query parameters correctly", async () => {
let response = await map({
url: "https://www.hfea.gov.uk",
- sitemapOnly: true,
+ sitemap: "only",
useMock: "map-query-params",
}, identity);
diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.ts
index f35143ba7..b9257f5cf 100644
--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts
+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts
@@ -19,7 +19,6 @@ describe("Parsers parameter tests", () => {
const response = await scrape({
url: pdfUrl,
parsers: ["pdf"],
- timeout: scrapeTimeout * 2,
}, identity);
expect(response.markdown).toBeDefined();
@@ -31,7 +30,6 @@ describe("Parsers parameter tests", () => {
const response = await scrape({
url: pdfUrl,
parsers: [],
- timeout: scrapeTimeout * 2,
}, identity);
expect(response.markdown).toBeDefined();
@@ -42,7 +40,6 @@ describe("Parsers parameter tests", () => {
const response = await scrape({
url: htmlUrl,
parsers: ["pdf"],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -53,7 +50,6 @@ describe("Parsers parameter tests", () => {
const response = await scrape({
url: htmlUrl,
parsers: [],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -66,7 +62,6 @@ describe("Parsers parameter tests", () => {
it.concurrent("parses PDF by default when parsers not specified", async () => {
const response = await scrape({
url: pdfUrl,
- timeout: scrapeTimeout * 2,
}, identity);
expect(response.markdown).toBeDefined();
@@ -80,7 +75,6 @@ describe("Parsers parameter tests", () => {
const raw = await scrapeRaw({
url: pdfUrl,
parsers: ["invalid-parser" as any],
- timeout: scrapeTimeout,
}, identity);
expect(raw.statusCode).toBe(400);
@@ -92,7 +86,6 @@ describe("Parsers parameter tests", () => {
const raw = await scrapeRaw({
url: pdfUrl,
parsers: "pdf" as any,
- timeout: scrapeTimeout,
}, identity);
expect(raw.statusCode).toBe(400);
@@ -104,7 +97,6 @@ describe("Parsers parameter tests", () => {
const raw = await scrapeRaw({
url: pdfUrl,
parsers: { pdf: true } as any,
- timeout: scrapeTimeout,
}, identity);
expect(raw.statusCode).toBe(400);
@@ -118,7 +110,6 @@ describe("Parsers parameter tests", () => {
const response = await scrape({
url: pdfUrl,
parsers: ["pdf"],
- timeout: scrapeTimeout * 2,
}, identity);
// Should bill based on number of pages when PDF parsing is enabled
@@ -129,7 +120,6 @@ describe("Parsers parameter tests", () => {
const response = await scrape({
url: pdfUrl,
parsers: [],
- timeout: scrapeTimeout * 2,
}, identity);
// Should bill flat rate (1 credit) when PDF parsing is disabled
diff --git a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts b/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts
index 2d9deb8bb..b3640d4a6 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts
@@ -20,7 +20,6 @@ describe("V2 Scrape Default maxAge", () => {
// First scrape to populate cache
const data1 = await scrape({
url,
- timeout: scrapeTimeout,
}, identity);
expect(data1).toBeDefined();
@@ -32,7 +31,6 @@ describe("V2 Scrape Default maxAge", () => {
// Second scrape should hit cache with default maxAge
const data2 = await scrape({
url,
- timeout: scrapeTimeout,
}, identity);
expect(data2).toBeDefined();
@@ -47,7 +45,6 @@ describe("V2 Scrape Default maxAge", () => {
// First scrape to populate cache
const data1 = await scrape({
url,
- timeout: scrapeTimeout,
}, identity);
expect(data1).toBeDefined();
@@ -59,7 +56,6 @@ describe("V2 Scrape Default maxAge", () => {
// Second scrape with maxAge=0 should miss cache
const data2 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -74,7 +70,6 @@ describe("V2 Scrape Default maxAge", () => {
// First scrape to populate cache
const data1 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: 3600000, // 1 hour in milliseconds
}, identity);
@@ -87,7 +82,6 @@ describe("V2 Scrape Default maxAge", () => {
// Second scrape with same maxAge should hit cache
const data2 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: 3600000, // 1 hour in milliseconds
}, identity);
diff --git a/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts b/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts
index 85e4fba9e..d82161eda 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts
@@ -16,7 +16,6 @@ describe("Scrape format variations", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: [{ type: "markdown" }],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -28,7 +27,6 @@ describe("Scrape format variations", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["markdown", "html", "links"],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -45,7 +43,6 @@ describe("Scrape format variations", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["screenshot"],
- timeout: scrapeTimeout,
}, identity);
expect(response.screenshot).toBeDefined();
@@ -58,7 +55,6 @@ describe("Scrape format variations", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: [{ type: "markdown" }],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -74,7 +70,6 @@ describe("Scrape format variations", () => {
{ type: "html" },
{ type: "links" }
],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -96,7 +91,6 @@ describe("Scrape format variations", () => {
quality: 80,
viewport: { width: 1920, height: 1080 }
}],
- timeout: scrapeTimeout,
}, identity);
expect(response.screenshot).toBeDefined();
@@ -113,7 +107,6 @@ describe("Scrape format variations", () => {
{ type: "html" },
"links"
],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -134,7 +127,6 @@ describe("Scrape format variations", () => {
},
{ type: "links" }
],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -160,7 +152,6 @@ describe("Scrape format variations", () => {
}
}
}],
- timeout: scrapeTimeout,
}, identity);
expect(response.json).toBeDefined();
@@ -180,7 +171,6 @@ describe("Scrape format variations", () => {
tag: "test-tag"
}
],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -193,7 +183,6 @@ describe("Scrape format variations", () => {
it.concurrent("default format is markdown when formats not specified", async () => {
const response = await scrape({
url: "https://firecrawl.dev",
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -204,7 +193,6 @@ describe("Scrape format variations", () => {
const raw = await scrapeRaw({
url: "https://firecrawl.dev",
formats: [{ type: "invalid-format" } as any],
- timeout: scrapeTimeout,
}, identity);
expect(raw.statusCode).toBe(400);
@@ -215,7 +203,6 @@ describe("Scrape format variations", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["markdown", "html", "rawHtml", "links"],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -230,7 +217,6 @@ describe("Scrape format variations", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["markdown", "html"],
- timeout: scrapeTimeout,
}, identity);
const keys = Object.keys(response);
@@ -243,7 +229,6 @@ describe("Scrape format variations", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: [{ type: "markdown" }, { type: "html" }],
- timeout: scrapeTimeout,
}, identity);
const keys = Object.keys(response);
@@ -256,19 +241,16 @@ describe("Scrape format variations", () => {
const response1 = await scrape({
url: "https://firecrawl.dev",
formats: ["markdown", "html"],
- timeout: scrapeTimeout,
}, identity);
const response2 = await scrape({
url: "https://firecrawl.dev",
formats: [{ type: "markdown" }, { type: "html" }],
- timeout: scrapeTimeout,
}, identity);
const response3 = await scrape({
url: "https://firecrawl.dev",
formats: ["markdown", { type: "html" }],
- timeout: scrapeTimeout,
}, identity);
expect(Object.keys(response1).sort()).toEqual(Object.keys(response2).sort());
diff --git a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts
index 658e5dcb4..ee202beee 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts
@@ -14,7 +14,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {
test("should default skipTlsVerification to true in v2 API", async () => {
const data = await scrape({
url: "https://expired.badssl.com/",
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -26,7 +25,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {
const response = await scrapeRaw({
url: "https://expired.badssl.com/",
skipTlsVerification: false,
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -41,7 +39,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {
test("should work with valid HTTPS sites regardless of skipTlsVerification setting", async () => {
const data = await scrape({
url: "https://firecrawl.dev",
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -54,7 +51,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {
const data = await scrape({
url: "https://firecrawl.dev",
formats: [{ type: "screenshot", fullPage: false }],
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -67,7 +63,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {
const data = await scrape({
url: "https://firecrawl.dev",
formats: [{ type: "screenshot", fullPage: true }],
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
diff --git a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
index 615dc5b5c..0432db45f 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
@@ -25,7 +25,6 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: scrapeTimeout,
}, identity);
expect(data).toBeDefined();
@@ -46,7 +45,6 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: scrapeTimeout,
}, identity);
expect(data).toBeDefined();
@@ -57,7 +55,6 @@ describe("V2 Scrape Screenshot Viewport", () => {
const data = await scrape({
url: "https://example.com",
formats: ["screenshot"],
- timeout: scrapeTimeout,
}, identity);
expect(data).toBeDefined();
@@ -73,7 +70,6 @@ describe("V2 Scrape Screenshot Viewport", () => {
fullPage: false
}
],
- timeout: scrapeTimeout,
}, identity);
expect(data).toBeDefined();
@@ -92,7 +88,6 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: scrapeTimeout,
}, identity);
expect(response.status).toBe(400);
@@ -111,7 +106,6 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: scrapeTimeout,
}, identity);
expect(response.status).toBe(400);
@@ -130,7 +124,6 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: scrapeTimeout,
}, identity);
expect(response.status).toBe(400);
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 27a91b116..55761395b 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -17,7 +17,6 @@ beforeAll(async () => {
await scrape({
url: changeTrackingTestUrl,
formats: ["markdown", "changeTracking"],
- timeout: scrapeTimeout,
}, identity);
}
}, 10000 + scrapeTimeout);
@@ -26,7 +25,6 @@ describe("Scrape tests", () => {
it.concurrent("works", async () => {
const response = await scrape({
url: "http://firecrawl.dev",
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toContain("Firecrawl");
@@ -101,14 +99,12 @@ describe("Scrape tests", () => {
// it.concurrent("works with Punycode domains", async () => {
// await scrape({
// url: "http://xn--1lqv92a901a.xn--ses554g/",
- // timeout: scrapeTimeout,
// }, identity);
// }, scrapeTimeout);
it.concurrent("handles non-UTF-8 encodings", async () => {
const response = await scrape({
url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toContain("ぐ け げ こ ご さ ざ し じ す ず せ ぜ そ ぞ た");
@@ -118,7 +114,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["links"],
- timeout: scrapeTimeout,
}, identity);
expect(response.links).toBeDefined();
@@ -129,7 +124,6 @@ describe("Scrape tests", () => {
it.concurrent("self-hosted proxy works", async () => {
const response = await scrape({
url: "https://icanhazip.com",
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);
@@ -139,7 +133,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://icanhazip.com",
waitFor: 100,
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);
@@ -151,7 +144,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "http://firecrawl.dev",
waitFor: 2000,
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toContain("Firecrawl");
@@ -163,7 +155,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://jsonplaceholder.typicode.com/todos/1",
formats: ["rawHtml"],
- timeout: scrapeTimeout,
}, identity);
const obj = JSON.parse(response.rawHtml!);
@@ -175,7 +166,6 @@ describe("Scrape tests", () => {
it.concurrent("scrape status works", async () => {
const response = await scrape({
url: "http://firecrawl.dev",
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toContain("Firecrawl");
@@ -192,7 +182,6 @@ describe("Scrape tests", () => {
await scrape({
url: "https://firecrawl.dev",
blockAds: true,
- timeout: scrapeTimeout,
}, identity);
}, scrapeTimeout);
@@ -200,7 +189,6 @@ describe("Scrape tests", () => {
await scrape({
url: "https://firecrawl.dev",
blockAds: false,
- timeout: scrapeTimeout,
}, identity);
}, scrapeTimeout);
});
@@ -214,7 +202,6 @@ describe("Scrape tests", () => {
url,
maxAge: scrapeTimeout * 3,
storeInCache: false,
- timeout: scrapeTimeout,
}, identity);
expect(response1.metadata.cacheState).toBe("miss");
@@ -224,7 +211,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
maxAge: scrapeTimeout * 3,
- timeout: scrapeTimeout,
}, identity);
expect(response2.metadata.cacheState).toBe("miss");
@@ -234,7 +220,6 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url,
maxAge: scrapeTimeout * 3,
- timeout: scrapeTimeout,
}, identity);
expect(response3.metadata.cacheState).toBe("hit");
@@ -243,7 +228,6 @@ describe("Scrape tests", () => {
const response4 = await scrape({
url,
maxAge: 1,
- timeout: scrapeTimeout,
}, identity);
expect(response4.metadata.cacheState).toBe("miss");
@@ -255,7 +239,6 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
}, identity);
@@ -265,7 +248,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
}, identity);
@@ -279,7 +261,6 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
formats: ["screenshot"],
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -290,7 +271,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
formats: ["screenshot"],
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
}, identity);
@@ -299,7 +279,6 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url,
formats: [{ type: "screenshot", fullPage: true }],
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3,
}, identity);
@@ -314,7 +293,6 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
formats: [{ type: "screenshot", fullPage: true }],
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -325,7 +303,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
formats: [{ type: "screenshot", fullPage: true }],
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
}, identity);
@@ -334,7 +311,6 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url,
formats: ["screenshot"],
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3,
}, identity);
@@ -349,14 +325,12 @@ describe("Scrape tests", () => {
await scrape({
url,
formats: ["markdown", "changeTracking"],
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
const response1 = await scrape({
url,
formats: ["markdown", "changeTracking"],
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
}, identity);
@@ -367,7 +341,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
formats: ["markdown"],
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3 + indexCooldown,
}, identity);
@@ -383,7 +356,6 @@ describe("Scrape tests", () => {
headers: {
"X-Test": "test",
},
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -391,7 +363,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2 + indexCooldown,
}, identity);
@@ -404,7 +375,6 @@ describe("Scrape tests", () => {
await scrape({
url,
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -412,7 +382,6 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
mobile: true,
}, identity);
@@ -423,7 +392,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3,
mobile: true,
}, identity);
@@ -437,7 +405,6 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout,
actions: [{
"type": "wait",
@@ -451,7 +418,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
}, identity);
@@ -464,7 +430,6 @@ describe("Scrape tests", () => {
await scrape({
url,
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -474,7 +439,6 @@ describe("Scrape tests", () => {
url,
location: { country: "DE" },
maxAge: scrapeTimeout * 2,
- timeout: scrapeTimeout,
}, identity);
expect(response1.metadata.cacheState).toBe("miss");
@@ -484,7 +448,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
location: { country: "DE" },
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3,
}, identity);
@@ -498,7 +461,6 @@ describe("Scrape tests", () => {
await scrape({
url,
blockAds: true,
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -507,7 +469,6 @@ describe("Scrape tests", () => {
const response0 = await scrape({
url,
blockAds: true,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2 + indexCooldown,
}, identity);
@@ -516,7 +477,6 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
blockAds: false,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3 + indexCooldown,
}, identity);
@@ -527,7 +487,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
blockAds: false,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 4 + 2 * indexCooldown,
}, identity);
@@ -541,7 +500,6 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url,
proxy: "stealth",
- timeout: scrapeTimeout,
maxAge: scrapeTimeout,
}, identity);
@@ -552,7 +510,6 @@ describe("Scrape tests", () => {
const response2 = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2 + indexCooldown,
}, identity);
@@ -561,7 +518,6 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url,
proxy: "stealth",
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 3 + indexCooldown,
}, identity);
@@ -574,7 +530,6 @@ describe("Scrape tests", () => {
await scrape({
url,
- timeout: scrapeTimeout,
maxAge: 0,
}, identity);
@@ -582,7 +537,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url,
- timeout: scrapeTimeout,
maxAge: scrapeTimeout * 2,
}, identity);
@@ -595,7 +549,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: changeTrackingTestUrl,
formats: ["markdown", "changeTracking"],
- timeout: scrapeTimeout,
}, identity);
expect(response.changeTracking).toBeDefined();
@@ -606,7 +559,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: changeTrackingTestUrl,
formats: ["markdown", { type: "changeTracking", modes: ["git-diff"] }],
- timeout: scrapeTimeout,
}, identity);
expect(response.changeTracking).toBeDefined();
@@ -624,7 +576,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: changeTrackingTestUrl,
formats: ["markdown", { type: "changeTracking", modes: ["json"], prompt: "Summarize the changes between the previous and current content" }],
- timeout: scrapeTimeout,
}, identity);
expect(response.changeTracking).toBeDefined();
@@ -661,7 +612,6 @@ describe("Scrape tests", () => {
}
}
],
- timeout: scrapeTimeout,
}, identity);
expect(response.changeTracking).toBeDefined();
@@ -691,7 +641,6 @@ describe("Scrape tests", () => {
}
}
}],
- timeout: scrapeTimeout * 2, // takes a while to run, LLM
}, identity);
expect(response.changeTracking).toBeDefined();
@@ -715,13 +664,11 @@ describe("Scrape tests", () => {
const response1 = await scrape({
url: changeTrackingTestUrl,
formats: ["markdown", { type: "changeTracking", tag: uuid1 }],
- timeout: scrapeTimeout,
}, identity);
const response2 = await scrape({
url: changeTrackingTestUrl,
formats: ["markdown", { type: "changeTracking", tag: uuid2 }],
- timeout: scrapeTimeout,
}, identity);
expect(response1.changeTracking?.previousScrapeAt).toBeNull();
@@ -732,7 +679,6 @@ describe("Scrape tests", () => {
const response3 = await scrape({
url: changeTrackingTestUrl,
formats: ["markdown", { type: "changeTracking", tag: uuid1 }],
- timeout: scrapeTimeout,
}, identity);
expect(response3.changeTracking?.previousScrapeAt).not.toBeNull();
@@ -744,7 +690,6 @@ describe("Scrape tests", () => {
it.concurrent("works without specifying an explicit location", async () => {
await scrape({
url: "https://iplocation.com",
- timeout: scrapeTimeout,
}, identity);
}, scrapeTimeout);
@@ -752,7 +697,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://iplocation.com",
location: { country: "US" },
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toContain("| Country | United States |");
@@ -764,7 +708,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "http://firecrawl.dev",
formats: ["screenshot"],
- timeout: scrapeTimeout,
}, identity);
expect(typeof response.screenshot).toBe("string");
@@ -774,7 +717,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "http://firecrawl.dev",
formats: [{ type: "screenshot", fullPage: true }],
- timeout: scrapeTimeout,
}, identity);
expect(typeof response.screenshot).toBe("string");
@@ -785,7 +727,6 @@ describe("Scrape tests", () => {
it.concurrent("works", async () => {
const response = await scrape({
url: "https://firecrawl.dev",
- timeout: scrapeTimeout,
actions: [{ type: "pdf" }],
}, identity);
@@ -800,7 +741,6 @@ describe("Scrape tests", () => {
it.concurrent("undefined works", async () => {
await scrape({
url: "http://firecrawl.dev",
- timeout: scrapeTimeout,
}, identity);
}, scrapeTimeout);
@@ -808,7 +748,6 @@ describe("Scrape tests", () => {
await scrape({
url: "http://firecrawl.dev",
proxy: "basic",
- timeout: scrapeTimeout,
}, identity);
}, scrapeTimeout);
@@ -816,7 +755,6 @@ describe("Scrape tests", () => {
await scrape({
url: "http://firecrawl.dev",
proxy: "stealth",
- timeout: scrapeTimeout * 2,
}, identity);
}, scrapeTimeout * 2);
@@ -824,7 +762,6 @@ describe("Scrape tests", () => {
const res = await scrape({
url: "http://firecrawl.dev",
proxy: "auto",
- timeout: scrapeTimeout * 2,
}, identity);
expect(res.metadata.proxyUsed).toBe("basic");
@@ -835,7 +772,6 @@ describe("Scrape tests", () => {
// const res = await scrape({
// url: "https://eo16f6718vph4un.m.pipedream.net", // always returns 403
// proxy: "auto",
- // timeout: scrapeTimeout * 2,
// }, identity);
// expect(res.metadata.proxyUsed).toBe("stealth");
@@ -855,17 +791,27 @@ describe("Scrape tests", () => {
it.concurrent("blocks long PDFs with insufficient timeout", async () => {
const response = await scrapeWithFailure({
url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",
- timeout: scrapeTimeout,
maxAge: 0,
+ timeout: 10000,
}, identity);
expect(response.error).toContain("Insufficient time to process PDF");
- }, scrapeTimeout);
+ }, 12000);
it.concurrent("scrapes long PDFs with sufficient timeout", async () => {
const response = await scrape({
url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",
+ maxAge: 0,
timeout: scrapeTimeout * 5,
+ }, identity);
+
+ // text on the last page
+ expect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");
+ }, scrapeTimeout * 5);
+
+ it.concurrent("scrapes long PDFs with default timeout", async () => {
+ const response = await scrape({
+ url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",
maxAge: 0,
}, identity);
@@ -876,7 +822,6 @@ describe("Scrape tests", () => {
it.concurrent("scrapes Google Docs links as PDFs", async () => {
const response = await scrape({
url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",
- timeout: scrapeTimeout * 5,
}, identity);
expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");
@@ -885,7 +830,6 @@ describe("Scrape tests", () => {
it.concurrent("scrapes Google Slides links as PDFs", async () => {
const response = await scrape({
url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view",
- timeout: scrapeTimeout * 5,
maxAge: 0,
}, identity);
@@ -914,7 +858,6 @@ describe("Scrape tests", () => {
},
required: ["company_mission", "supports_sso", "is_open_source"],
} }],
- timeout: scrapeTimeout,
}, identity);
expect(response).toHaveProperty("json");
@@ -934,7 +877,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["summary"],
- timeout: scrapeTimeout,
}, identity);
expect(response.summary).toBeDefined();
@@ -946,7 +888,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["markdown", "summary"],
- timeout: scrapeTimeout,
}, identity);
expect(response.summary).toBeDefined();
@@ -966,7 +907,6 @@ describe("Scrape tests", () => {
}
} }
}],
- timeout: scrapeTimeout,
}, identity);
expect(response.summary).toBeDefined();
@@ -978,7 +918,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://firecrawl.dev",
formats: ["markdown", "html", "summary"],
- timeout: scrapeTimeout,
}, identity);
expect(response.summary).toBeDefined();
@@ -992,7 +931,6 @@ describe("Scrape tests", () => {
it.concurrent("sourceURL stays unnormalized", async () => {
const response = await scrape({
url: "https://firecrawl.dev/?pagewanted=all&et_blog",
- timeout: scrapeTimeout,
}, identity);
expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");
@@ -1002,7 +940,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://jsonplaceholder.typicode.com/todos/1",
formats: ["markdown"],
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toContain("```json");
@@ -1013,7 +950,6 @@ describe("Scrape tests", () => {
const response = await scrape({
url: "https://httpbin.org/html",
__experimental_omceDomain: "fake-domain.com",
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
@@ -1025,7 +961,6 @@ describe("Scrape tests", () => {
url: "https://httpbin.org/html",
__experimental_omceDomain: "test-domain.org",
__experimental_omce: true,
- timeout: scrapeTimeout,
}, identity);
expect(response.markdown).toBeDefined();
diff --git a/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.ts b/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.ts
index 3acc5ede6..29311a469 100644
--- a/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.ts
+++ b/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.ts
@@ -27,7 +27,6 @@ describe("V2 System Prompt Rejection", () => {
systemPrompt: "Custom system prompt that should be rejected",
},
],
- timeout: scrapeTimeout + 30000,
} as any,
identity,
);
@@ -51,7 +50,6 @@ describe("V2 System Prompt Rejection", () => {
prompt: "Extract the title",
},
],
- timeout: scrapeTimeout + 30000,
},
identity,
);
diff --git a/apps/api/src/controllers/v2/credit-usage.ts b/apps/api/src/controllers/v2/credit-usage.ts
new file mode 100644
index 000000000..794861a58
--- /dev/null
+++ b/apps/api/src/controllers/v2/credit-usage.ts
@@ -0,0 +1,46 @@
+import { Request, Response } from "express";
+import { RequestWithAuth } from "./types";
+import { getACUCTeam } from "../auth";
+import { logger } from "../../lib/logger";
+import { RateLimiterMode } from "../../types";
+
+export async function creditUsageController(
+ req: RequestWithAuth,
+ res: Response,
+): Promise<void> {
+ try {
+ // If we already have the credit usage info from auth, use it
+ if (req.acuc) {
+ res.json({
+ success: true,
+ data: {
+ remaining_credits: req.acuc.remaining_credits,
+ },
+ });
+ return;
+ }
+
+ // Otherwise fetch fresh data
+ const chunk = await getACUCTeam(req.auth.team_id, false, false, RateLimiterMode.Scrape);
+ if (!chunk) {
+ res.status(404).json({
+ success: false,
+ error: "Could not find credit usage information",
+ });
+ return;
+ }
+
+ res.json({
+ success: true,
+ data: {
+ remaining_credits: chunk.remaining_credits,
+ },
+ });
+ } catch (error) {
+ logger.error("Error in credit usage controller:", error);
+ res.status(500).json({
+ success: false,
+ error: "Internal server error while fetching credit usage",
+ });
+ }
+}
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index eb0941933..87d2577b0 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -324,9 +324,12 @@ export async function mapController(
url: req.body.url,
search: req.body.search,
limit: req.body.limit,
- ignoreSitemap: req.body.ignoreSitemap,
+ ignoreSitemap: req.body.sitemap === "skip",
includeSubdomains: req.body.includeSubdomains,
- crawlerOptions: req.body,
+ crawlerOptions: {
+ ...req.body,
+ sitemapOnly: req.body.sitemap === "only",
+ },
origin: req.body.origin,
teamId: req.auth.team_id,
abort: abort.signal,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index a9542e55e..df7cf2220 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -97,7 +97,7 @@ export async function scrapeController(
let doc: Document;
try {
- doc = await waitForJob(jobId, timeout + totalWait);
+ doc = await waitForJob(jobId, (timeout !== undefined) ? timeout + totalWait : null);
} catch (e) {
logger.error(`Error in scrapeController`, {
startTime,
diff --git a/apps/api/src/controllers/v2/token-usage.ts b/apps/api/src/controllers/v2/token-usage.ts
new file mode 100644
index 000000000..1e4f6116c
--- /dev/null
+++ b/apps/api/src/controllers/v2/token-usage.ts
@@ -0,0 +1,46 @@
+import { Request, Response } from "express";
+import { RequestWithAuth } from "./types";
+import { getACUC, getACUCTeam } from "../auth";
+import { logger } from "../../lib/logger";
+import { RateLimiterMode } from "../../types";
+
+export async function tokenUsageController(
+ req: RequestWithAuth,
+ res: Response,
+): Promise<void> {
+ try {
+ // If we already have the token usage info from auth, use it
+ if (req.acuc) {
+ res.json({
+ success: true,
+ data: {
+ remaining_tokens: req.acuc.remaining_credits,
+ },
+ });
+ return;
+ }
+
+ // Otherwise fetch fresh data
+ const chunk = await getACUCTeam(req.auth.team_id, false, false, RateLimiterMode.Extract);
+ if (!chunk) {
+ res.status(404).json({
+ success: false,
+ error: "Could not find token usage information",
+ });
+ return;
+ }
+
+ res.json({
+ success: true,
+ data: {
+ remaining_tokens: chunk.remaining_credits,
+ },
+ });
+ } catch (error) {
+ logger.error("Error in token usage controller:", error);
+ res.status(500).json({
+ success: false,
+ error: "Internal server error while fetching token usage",
+ });
+ }
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 76abebcad..ee6c80a30 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -425,7 +425,7 @@ export const extractV1Options = z
origin: z.string().optional().default("api"),
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
urlTrace: z.boolean().default(false),
- timeout: z.number().int().positive().finite().safe().default(60000),
+ timeout: z.number().int().positive().finite().safe().optional(),
__experimental_streamSteps: z.boolean().default(false),
__experimental_llmUsage: z.boolean().default(false),
__experimental_showSources: z.boolean().default(false),
@@ -463,12 +463,10 @@ export type ExtractRequest = z.infer<typeof extractRequestSchema>;
export type ExtractRequestInput = z.input<typeof extractRequestSchema>;
export const scrapeRequestSchema = baseScrapeOptions
- .omit({ timeout: true })
.extend({
url,
origin: z.string().optional().default("api"),
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
- timeout: z.number().int().positive().finite().safe().default(30000),
zeroDataRetention: z.boolean().optional(),
})
.strict(strictMessage)
@@ -603,14 +601,14 @@ export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
export const mapRequestSchema = crawlerOptions
+ .omit({ ignoreSitemap: true })
.extend({
url,
origin: z.string().optional().default("api"),
integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),
includeSubdomains: z.boolean().default(true),
search: z.string().optional(),
- ignoreSitemap: z.boolean().default(false),
- sitemapOnly: z.boolean().default(false),
+ sitemap: z.enum(["only", "include", "skip"]).default("include"),
limit: z.number().min(1).max(30000).default(5000),
timeout: z.number().positive().finite().optional(),
useMock: z.string().optional(),
diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
index 6b6af2c60..60a484bf7 100644
--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts
@@ -12,7 +12,6 @@ import { generateObject, generateText, LanguageModel } from 'ai';
import { jsonSchema } from 'ai';
import { getModel } from "../../../lib/generic-ai";
import { z } from "zod";
-import { EngineResultsTracker, Meta } from "../../../scraper/scrapeURL";
// Get max tokens from model prices
const getModelLimits_F0 = (model: string) => {
@@ -34,7 +33,6 @@ const getModelLimits_F0 = (model: string) => {
export class LLMRefusalError extends Error {
public refusal: string;
- public results: EngineResultsTracker | undefined;
constructor(refusal: string) {
super("LLM refused to extract the website's content");
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index d89a51783..118da61f0 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -10,7 +10,6 @@ import { supabase_service } from "../services/supabase";
import { logger as _logger } from "../lib/logger";
import { configDotenv } from "dotenv";
import {
- EngineResultsTracker,
scrapeURL,
ScrapeUrlResponse,
} from "../scraper/scrapeURL";
@@ -76,7 +75,6 @@ export async function runWebScraper({
logger.info("runWebScraper called");
let response: ScrapeUrlResponse | undefined = undefined;
- let engines: EngineResultsTracker = {};
let error: any = undefined;
for (let i = 0; i < tries; i++) {
@@ -90,7 +88,6 @@ export async function runWebScraper({
}
response = undefined;
- engines = {};
error = undefined;
try {
@@ -116,8 +113,6 @@ export async function runWebScraper({
}
}
- engines = response.engines;
-
if (
(response.document.metadata.statusCode >= 200 &&
response.document.metadata.statusCode < 300) ||
@@ -128,12 +123,6 @@ export async function runWebScraper({
}
} catch (_error) {
error = _error;
- engines =
- response !== undefined
- ? response.engines
- : typeof error === "object" && error !== null
- ? ((error as any).results ?? {})
- : {};
}
}
@@ -180,8 +169,6 @@ export async function runWebScraper({
return {
success: false,
error,
- logs: ["no logs -- error coming from runWebScraper"],
- engines,
};
}
}
@@ -191,7 +178,6 @@ const saveJob = async (
job: Job,
result: any,
mode: string,
- engines?: EngineResultsTracker,
) => {
try {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index a9c992ebb..acfdb128c 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -11,6 +11,8 @@ import { mapController } from "../controllers/v2/map";
import { crawlErrorsController } from "../controllers/v2/crawl-errors";
import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";
import { scrapeStatusController } from "../controllers/v2/scrape-status";
+import { creditUsageController } from "../controllers/v2/credit-usage";
+import { tokenUsageController } from "../controllers/v2/token-usage";
import {
authMiddleware,
checkCreditsMiddleware,
@@ -96,3 +98,15 @@ v2Router.get(
authMiddleware(RateLimiterMode.CrawlStatus),
wrap(crawlErrorsController),
);
+
+v2Router.get(
+ "/credit-usage",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(creditUsageController),
+);
+
+v2Router.get(
+ "/token-usage",
+ authMiddleware(RateLimiterMode.ExtractStatus),
+ wrap(tokenUsageController),
+);
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 08a0848dc..c4429431e 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -53,7 +53,13 @@ export async function getLinksFromSitemap(
...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const] : []),
],
v0DisableJsDom: true,
- abort,
+ externalAbort: abort ? {
+ signal: abort,
+ tier: "external",
+ throwable() {
+ return new Error("Sitemap fetch aborted");
+ },
+ } : undefined,
teamId: "sitemap",
zeroDataRetention,
},
diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
index 11c7304a2..eb27b3bbc 100644
--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts
@@ -3,10 +3,10 @@ import { EngineScrapeResult } from "..";
import { downloadFile } from "../utils/downloadFile";
import mammoth from "mammoth";
-export async function scrapeDOCX(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {
+export async function scrapeDOCX(meta: Meta): Promise<EngineScrapeResult> {
const { response, tempFilePath } = await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {
headers: meta.options.headers,
- signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeToRun ?? 300000),
+ signal: meta.abort.asSignal(),
});
return {
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index e0e31c8cb..1ab8baca9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -1,7 +1,7 @@
import * as undici from "undici";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
-import { SSLError, TimeoutError } from "../../error";
+import { SSLError } from "../../error";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import {
InsecureConnectionError,
@@ -13,10 +13,7 @@ import { TextDecoder } from "util";
export async function scrapeURLWithFetch(
meta: Meta,
- timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
- const timeout = timeToRun ?? 300000;
-
const mockOptions = {
url: meta.rewrittenUrl ?? meta.url,
@@ -55,23 +52,12 @@ export async function scrapeURLWithFetch(
};
} else {
try {
- const x = await Promise.race([
- undici.fetch(meta.rewrittenUrl ?? meta.url, {
- dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,
- redirect: "follow",
- headers: meta.options.headers,
- signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
- }),
- (async () => {
- await new Promise((resolve) =>
- setTimeout(() => resolve(null), timeout),
- );
- throw new TimeoutError(
- "Fetch was unable to scrape the page before timing out",
- { cause: { timeout } },
- );
- })(),
- ]);
+ const x = await undici.fetch(meta.rewrittenUrl ?? meta.url, {
+ dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,
+ redirect: "follow",
+ headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
+ });
const buf = Buffer.from(await x.arrayBuffer());
let text = buf.toString("utf8");
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
index 5fd30ff38..71ec86c24 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts
@@ -19,7 +19,6 @@ import {
DNSResolutionError,
SiteError,
SSLError,
- TimeoutError,
UnsupportedFileError,
FEPageLoadFailed,
} from "../../error";
@@ -28,8 +27,9 @@ import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import { fireEngineDelete } from "./delete";
import { MockState } from "../../lib/mock";
import { getInnerJSON } from "../../../../lib/html-transformer";
-import { hasFormatOfType, hasAnyFormatOfTypes } from "../../../../lib/format-utils";
-import { Action, TimeoutSignal } from "../../../../controllers/v1/types";
+import { hasFormatOfType } from "../../../../lib/format-utils";
+import { Action } from "../../../../controllers/v1/types";
+import { AbortManagerThrownError } from "../../lib/abortManager";
// This function does not take `Meta` on purpose. It may not access any
// meta values to construct the request -- that must be done by the
@@ -43,7 +43,6 @@ async function performFireEngineScrape<
meta: Meta,
logger: Logger,
request: FireEngineScrapeRequestCommon & Engine,
- timeout: number,
mock: MockState | null,
abort?: AbortSignal,
production = true,
@@ -56,13 +55,11 @@ async function performFireEngineScrape<
production,
);
- const startTime = Date.now();
const errorLimit = 3;
let errors: any[] = [];
let status: FireEngineCheckStatusSuccess | undefined = undefined;
while (status === undefined) {
- abort?.throwIfAborted();
if (errors.length >= errorLimit) {
logger.error("Error limit hit.", { errors });
fireEngineDelete(
@@ -80,16 +77,7 @@ async function performFireEngineScrape<
});
}
- if (Date.now() - startTime > timeout) {
- logger.info(
- "Fire-engine was unable to scrape the page before timing out.",
- { errors, timeout },
- );
- throw new TimeoutError(
- "Fire-engine was unable to scrape the page before timing out",
- { cause: { errors, timeout } },
- );
- }
+ meta.abort.throwIfAborted();
try {
status = await fireEngineCheckStatus(
@@ -127,7 +115,7 @@ async function performFireEngineScrape<
jobId: scrape.jobId,
});
throw error;
- } else if (error instanceof TimeoutSignal) {
+ } else if (error instanceof AbortManagerThrownError) {
fireEngineDelete(
logger.child({
method: "performFireEngineScrape/fireEngineDelete",
@@ -138,7 +126,7 @@ async function performFireEngineScrape<
undefined,
production,
);
- throw new TimeoutError("Fire-engine timed out while checking status", { cause: { errors, timeout } });
+ throw error;
} else {
errors.push(error);
logger.debug(
@@ -189,7 +177,6 @@ async function performFireEngineScrape<
export async function scrapeURLWithFireEngineChromeCDP(
meta: Meta,
- timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
const actions: Action[] = [
// Transform waitFor option into an action (unsupported by chrome-cdp)
@@ -223,8 +210,6 @@ export async function scrapeURLWithFireEngineChromeCDP(
0,
);
- const timeout = (timeToRun ?? 300000) + totalWait;
-
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestChromeCDP = {
url: meta.rewrittenUrl ?? meta.url,
@@ -240,7 +225,7 @@ export async function scrapeURLWithFireEngineChromeCDP(
priority: meta.internalOptions.priority,
geolocation: meta.options.location,
mobile: meta.options.mobile,
- timeout, // TODO: better timeout logic
+ timeout: meta.abort.scrapeTimeout() ?? 300000,
disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,
mobileProxy: meta.featureFlags.has("stealthProxy"),
saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS,
@@ -254,9 +239,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
request,
}),
request,
- timeout,
meta.mock,
- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
+ meta.abort.asSignal(),
true,
);
@@ -311,10 +295,8 @@ export async function scrapeURLWithFireEngineChromeCDP(
export async function scrapeURLWithFireEnginePlaywright(
meta: Meta,
- timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
const totalWait = meta.options.waitFor;
- const timeout = (timeToRun ?? 300000) + totalWait;
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestPlaywright = {
@@ -331,7 +313,7 @@ export async function scrapeURLWithFireEnginePlaywright(
blockAds: meta.options.blockAds,
mobileProxy: meta.featureFlags.has("stealthProxy"),
- timeout,
+ timeout: meta.abort.scrapeTimeout() ?? 300000,
saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
};
@@ -343,9 +325,8 @@ export async function scrapeURLWithFireEnginePlaywright(
request,
}),
request,
- timeout,
meta.mock,
- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
+ meta.abort.asSignal(),
);
if (!response.url) {
@@ -378,10 +359,7 @@ export async function scrapeURLWithFireEnginePlaywright(
export async function scrapeURLWithFireEngineTLSClient(
meta: Meta,
- timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
- const timeout = timeToRun ?? 30000;
-
const request: FireEngineScrapeRequestCommon &
FireEngineScrapeRequestTLSClient = {
url: meta.rewrittenUrl ?? meta.url,
@@ -396,7 +374,7 @@ export async function scrapeURLWithFireEngineTLSClient(
disableJsDom: meta.internalOptions.v0DisableJsDom,
mobileProxy: meta.featureFlags.has("stealthProxy"),
- timeout,
+ timeout: meta.abort.scrapeTimeout() ?? 300000,
saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS,
zeroDataRetention: meta.internalOptions.zeroDataRetention,
};
@@ -408,9 +386,8 @@ export async function scrapeURLWithFireEngineTLSClient(
request,
}),
request,
- timeout,
meta.mock,
- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
+ meta.abort.asSignal(),
);
if (!response.url) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index bc734e010..9e4d380a8 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -127,7 +127,6 @@ export type EngineScrapeResult = {
const engineHandlers: {
[E in Engine]: (
meta: Meta,
- timeToRun: number | undefined,
) => Promise<EngineScrapeResult>;
} = {
index: scrapeURLWithIndex,
@@ -513,7 +512,6 @@ export function buildFallbackList(meta: Meta): {
export async function scrapeURLWithEngine(
meta: Meta,
engine: Engine,
- timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
const fn = engineHandlers[engine];
const logger = meta.logger.child({
@@ -525,5 +523,5 @@ export async function scrapeURLWithEngine(
logger,
};
- return await fn(_meta, timeToRun);
+ return await fn(_meta);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
index 789e0b637..769350b26 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts
@@ -2,7 +2,7 @@ import { Document } from "../../../../controllers/v1/types";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob, addDomainFrequencyJob } from "../../../../services";
-import { EngineError, IndexMissError, TimeoutError } from "../../error";
+import { EngineError, IndexMissError } from "../../error";
import crypto from "crypto";
export async function sendDocumentToIndex(meta: Meta, document: Document) {
@@ -136,7 +136,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {
const errorCountToRegister = 3;
-export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {
+export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult> {
const normalizedURL = normalizeURLForIndex(meta.url);
const urlHash = hashURL(normalizedURL);
@@ -165,14 +165,9 @@ export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefin
selector = selector.is("location_languages", null);
}
- const { data, error } = await Promise.race([
- selector
- .order("created_at", { ascending: false })
- .limit(5),
- new Promise<{ data: { id: any; created_at: any; status: any }[], error: any }>((resolve, reject) => {
- setTimeout(() => reject(new TimeoutError()), timeToRun ?? 10000);
- }),
- ]);
+ const { data, error } = await selector
+ .order("created_at", { ascending: false })
+ .limit(5);
if (error || !data) {
throw new EngineError("Failed to retrieve URL from DB index", {
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 177104b3c..f63ddaaeb 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -12,13 +12,13 @@ import {
PDFInsufficientTimeError,
PDFPrefetchFailed,
RemoveFeatureError,
- TimeoutError,
} from "../../error";
import { readFile, unlink } from "node:fs/promises";
import path from "node:path";
import type { Response } from "undici";
import { getPageCount } from "../../../../lib/pdf-parser";
import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";
+import { AbortManagerThrownError } from "../../lib/abortManager";
type PDFProcessorResult = { html: string; markdown?: string };
@@ -28,15 +28,12 @@ const MILLISECONDS_PER_PAGE = 150;
async function scrapePDFWithRunPodMU(
meta: Meta,
tempFilePath: string,
- timeToRun: number | undefined,
base64Content: string,
): Promise<PDFProcessorResult> {
meta.logger.debug("Processing PDF document with RunPod MU", {
tempFilePath,
});
- const preCacheCheckStartTime = Date.now();
-
try {
const cachedResult = await getPdfResultFromCache(base64Content);
if (cachedResult) {
@@ -52,14 +49,7 @@ async function scrapePDFWithRunPodMU(
});
}
- const timeout = timeToRun
- ? timeToRun - (Date.now() - preCacheCheckStartTime)
- : undefined;
- if (timeout && timeout < 0) {
- throw new TimeoutError("MU PDF parser already timed out before call");
- }
-
- const abort = timeout ? AbortSignal.timeout(timeout) : undefined;
+ meta.abort.throwIfAborted();
const podStart = await robustFetch({
url:
@@ -72,7 +62,7 @@ async function scrapePDFWithRunPodMU(
input: {
file_content: base64Content,
filename: path.basename(tempFilePath) + ".pdf",
- timeout,
+ timeout: meta.abort.scrapeTimeout(),
created_at: Date.now(),
},
},
@@ -89,19 +79,17 @@ async function scrapePDFWithRunPodMU(
.optional(),
}),
mock: meta.mock,
- abort,
+ abort: meta.abort.asSignal(),
});
-
-
let status: string = podStart.status;
let result: { markdown: string } | undefined = podStart.output;
if (status === "IN_QUEUE" || status === "IN_PROGRESS") {
do {
- abort?.throwIfAborted();
+ meta.abort.throwIfAborted();
await new Promise((resolve) => setTimeout(resolve, 2500));
- abort?.throwIfAborted();
+ meta.abort.throwIfAborted();
const podStatus = await robustFetch({
url: `https://api.runpod.ai/v2/${process.env.RUNPOD_MU_POD_ID}/status/${podStart.id}`,
method: "GET",
@@ -120,7 +108,7 @@ async function scrapePDFWithRunPodMU(
.optional(),
}),
mock: meta.mock,
- abort,
+ abort: meta.abort.asSignal(),
});
status = podStatus.status;
result = podStatus.output;
@@ -171,10 +159,7 @@ async function scrapePDFWithParsePDF(
export async function scrapePDF(
meta: Meta,
- timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
- const startTime = Date.now();
-
const shouldParsePDF = meta.options.parsers?.includes("pdf") ?? true;
if (!shouldParsePDF) {
@@ -241,7 +226,7 @@ export async function scrapePDF(
}
const pageCount = await getPageCount(tempFilePath);
- if (pageCount * MILLISECONDS_PER_PAGE > (timeToRun ?? Infinity)) {
+ if (pageCount * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {
throw new PDFInsufficientTimeError(
pageCount,
pageCount * MILLISECONDS_PER_PAGE + 5000,
@@ -252,8 +237,6 @@ export async function scrapePDF(
const base64Content = (await readFile(tempFilePath)).toString("base64");
- const remainingTime = timeToRun ? timeToRun - (Date.now() - startTime) : undefined;
-
// First try RunPod MU if conditions are met
if (
base64Content.length < MAX_FILE_SIZE &&
@@ -269,28 +252,27 @@ export async function scrapePDF(
}),
},
tempFilePath,
- remainingTime,
base64Content,
);
} catch (error) {
if (
error instanceof RemoveFeatureError ||
- error instanceof TimeoutError
+ error instanceof AbortManagerThrownError
) {
throw error;
- } else if (
- (error instanceof Error && error.name === "TimeoutError") ||
- (error instanceof Error &&
- error.message === "Request failed" &&
- error.cause &&
- error.cause instanceof Error &&
- error.cause.name === "TimeoutError") ||
- (error instanceof Error && error.name === "TimeoutSignal")
- ) {
- meta.logger.warn("RunPod MU timed out", { remainingTime });
- throw new TimeoutError(
- "PDF parsing timed out, please increase the timeout parameter in your scrape request",
- );
+ // } else if (
+ // (error instanceof Error && error.name === "TimeoutError") ||
+ // (error instanceof Error &&
+ // error.message === "Request failed" &&
+ // error.cause &&
+ // error.cause instanceof Error &&
+ // error.cause.name === "TimeoutError") ||
+ // (error instanceof Error && error.name === "TimeoutSignal")
+ // ) {
+ // meta.logger.warn("RunPod MU timed out", { timeout: remainingTime });
+ // throw new TimeoutError(
+ // "PDF parsing timed out, please increase the timeout parameter in your scrape request",
+ // );
}
meta.logger.warn(
"RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
index 98285543b..dd0fc29fc 100644
--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -1,48 +1,35 @@
import { z } from "zod";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
-import { TimeoutError } from "../../error";
import { robustFetch } from "../../lib/fetch";
import { getInnerJSON } from "../../../../lib/html-transformer";
export async function scrapeURLWithPlaywright(
meta: Meta,
- timeToRun: number | undefined,
): Promise<EngineScrapeResult> {
- const timeout = (timeToRun ?? 300000) + meta.options.waitFor;
-
- const response = await Promise.race([
- await robustFetch({
- url: process.env.PLAYWRIGHT_MICROSERVICE_URL!,
- headers: {
- "Content-Type": "application/json",
- },
- body: {
- url: meta.rewrittenUrl ?? meta.url,
- wait_after_load: meta.options.waitFor,
- timeout,
- headers: meta.options.headers,
- skip_tls_verification: meta.options.skipTlsVerification,
- },
- method: "POST",
- logger: meta.logger.child("scrapeURLWithPlaywright/robustFetch"),
- schema: z.object({
- content: z.string(),
- pageStatusCode: z.number(),
- pageError: z.string().optional(),
- contentType: z.string().optional(),
- }),
- mock: meta.mock,
- abort: AbortSignal.timeout(timeout),
+ const response = await robustFetch({
+ url: process.env.PLAYWRIGHT_MICROSERVICE_URL!,
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: {
+ url: meta.rewrittenUrl ?? meta.url,
+ wait_after_load: meta.options.waitFor,
+ timeout: meta.abort.scrapeTimeout(),
+ headers: meta.options.headers,
+ skip_tls_verification: meta.options.skipTlsVerification,
+ },
+ method: "POST",
+ logger: meta.logger.child("scrapeURLWithPlaywright/robustFetch"),
+ schema: z.object({
+ content: z.string(),
+ pageStatusCode: z.number(),
+ pageError: z.string().optional(),
+ contentType: z.string().optional(),
}),
- (async () => {
- await new Promise((resolve) => setTimeout(() => resolve(null), timeout));
- throw new TimeoutError(
- "Playwright was unable to scrape the page before timing out",
- { cause: { timeout } },
- );
- })(),
- ]);
+ mock: meta.mock,
+ abort: meta.abort.asSignal(),
+ });
if (response.contentType?.includes("application/json")) {
response.content = await getInnerJSON(response.content);
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index b2ac60aaa..295b208b0 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -1,4 +1,4 @@
-import { EngineResultsTracker, Meta } from ".";
+import { Meta } from ".";
import { Engine, FeatureFlag } from "./engines";
export class EngineError extends Error {
@@ -7,22 +7,14 @@ export class EngineError extends Error {
}
}
-export class TimeoutError extends Error {
- constructor(message?: string, options?: ErrorOptions) {
- super(message, options);
- }
-}
-
export class NoEnginesLeftError extends Error {
public fallbackList: Engine[];
- public results: EngineResultsTracker;
- constructor(fallbackList: Engine[], results: EngineResultsTracker) {
+ constructor(fallbackList: Engine[]) {
super(
"All scraping engines failed! -- Double check the URL to make sure it's not broken. If the issue persists, contact us at [email protected].",
);
this.fallbackList = fallbackList;
- this.results = results;
}
}
@@ -127,3 +119,41 @@ export class FEPageLoadFailed extends Error {
super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request.");
}
}
+
+export class ScrapeURLScopedTimeoutError extends Error {
+ constructor(message: string, options?: ErrorOptions) {
+ super(message, options);
+ }
+}
+
+export class ScrapeTimeoutError extends ScrapeURLScopedTimeoutError {
+ name = "ScrapeTimeoutError";
+
+ constructor() {
+ super("Scrape timed out");
+ }
+}
+
+export class EngineSnipedError extends ScrapeURLScopedTimeoutError {
+ name = "EngineSnipedError";
+
+ constructor() {
+ super("Engine got sniped");
+ }
+}
+
+export class EngineUnsuccessfulError extends Error {
+ name = "EngineUnsuccessfulError";
+
+ constructor(engine: Engine) {
+ super(`Engine ${engine} was unsuccessful`);
+ }
+}
+
+export class WaterfallNextEngineSignal extends Error {
+ name = "WaterfallNextEngineSignal";
+
+ constructor() {
+ super("Waterfall next engine");
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 25088db8c..bb22a2875 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -21,7 +21,6 @@ import {
PDFAntibotError,
RemoveFeatureError,
SiteError,
- TimeoutError,
UnsupportedFileError,
SSLError,
PDFInsufficientTimeError,
@@ -30,6 +29,10 @@ import {
ZDRViolationError,
PDFPrefetchFailed,
FEPageLoadFailed,
+ ScrapeTimeoutError,
+ EngineSnipedError,
+ WaterfallNextEngineSignal,
+ EngineUnsuccessfulError,
} from "./error";
import { executeTransformers } from "./transformers";
import { LLMRefusalError } from "./transformers/llmExtract";
@@ -39,6 +42,7 @@ import { CostTracking } from "../../lib/extract/extraction-service";
import { robustFetch } from "./lib/fetch";
import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";
import { checkRobotsTxt } from "../../lib/robots-txt";
+import { AbortInstance, AbortManager, AbortManagerThrownError } from "./lib/abortManager";
export type ScrapeUrlResponse = (
| {
@@ -49,10 +53,7 @@ export type ScrapeUrlResponse = (
success: false;
error: any;
}
-) & {
- logs: any[];
- engines: EngineResultsTracker;
-};
+);
export type Meta = {
id: string;
@@ -61,7 +62,7 @@ export type Meta = {
options: ScrapeOptions;
internalOptions: InternalOptions;
logger: Logger;
- logs: any[];
+ abort: AbortManager;
featureFlags: Set<FeatureFlag>;
mock: MockState | null;
pdfPrefetch: {
@@ -72,7 +73,6 @@ export type Meta = {
} | null | undefined; // undefined: no prefetch yet, null: prefetch came back empty
costTracking: CostTracking;
winnerEngine?: Engine;
- results: EngineResultsTracker;
};
function buildFeatureFlags(
@@ -188,7 +188,6 @@ async function buildMetaObject(
team_id: internalOptions.teamId,
crawlId: internalOptions.crawlId,
});
- const logs: any[] = [];
return {
id,
@@ -197,7 +196,17 @@ async function buildMetaObject(
options,
internalOptions,
logger,
- logs,
+ abort: new AbortManager(
+ internalOptions.externalAbort,
+ options.timeout !== undefined ? {
+ signal: AbortSignal.timeout(options.timeout),
+ tier: "scrape",
+ timesOutAt: new Date(Date.now() + options.timeout),
+ throwable() {
+ return new ScrapeTimeoutError();
+ },
+ } : undefined,
+ ),
featureFlags: buildFeatureFlags(url, options, internalOptions),
mock:
options.useMock !== undefined
@@ -205,7 +214,6 @@ async function buildMetaObject(
: null,
pdfPrefetch: undefined,
costTracking,
- results: {},
};
}
@@ -221,7 +229,7 @@ export type InternalOptions = {
v0DisableJsDom?: boolean;
disableSmartWaitCache?: boolean; // Passed along to fire-engine
isBackgroundIndex?: boolean;
- abort?: AbortSignal;
+ externalAbort?: AbortInstance;
urlInvisibleInCurrentCrawl?: boolean;
unnormalizedSourceURL?: string;
@@ -236,29 +244,6 @@ export type InternalOptions = {
v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility
};
-export type EngineResultsTracker = {
- [E in Engine]?: (
- | {
- state: "error";
- error: any;
- unexpected: boolean;
- }
- | {
- state: "success";
- result: EngineScrapeResult & { markdown: string };
- factors: Record<string, boolean>;
- unsupportedFeatures: Set<FeatureFlag>;
- }
- | {
- state: "timeout";
- error: any;
- }
- ) & {
- startedAt: number;
- finishedAt: number;
- };
-};
-
export type EngineScrapeResultWithContext = {
engine: Engine;
unsupportedFeatures: Set<FeatureFlag>;
@@ -275,6 +260,58 @@ function safeguardCircularError<T>(error: T): T {
}
}
+async function scrapeURLLoopIter(meta: Meta, engine: Engine, snipeAbort): Promise<EngineScrapeResult & { markdown: string }> {
+ const _engineResult = await scrapeURLWithEngine({
+ ...meta,
+ abort: meta.abort.child(snipeAbort),
+ }, engine);
+
+ if (_engineResult.markdown === undefined) {
+ _engineResult.markdown = await parseMarkdown(_engineResult.html);
+ }
+
+ const engineResult = _engineResult as EngineScrapeResult & {
+ markdown: string;
+ };
+
+ // Success factors
+ const isLongEnough = engineResult.markdown.length > 0;
+ const isGoodStatusCode =
+ (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||
+ engineResult.statusCode === 304;
+ const hasNoPageError = engineResult.error === undefined;
+ const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);
+
+ if (isLikelyProxyError && meta.options.proxy === "auto" && !meta.featureFlags.has("stealthProxy")) {
+ meta.logger.info("Scrape via " + engine + " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.");
+ throw new AddFeatureError(["stealthProxy"]);
+ }
+
+ // NOTE: TODO: what to do when status code is bad is tough...
+ // we cannot just rely on text because error messages can be brief and not hit the limit
+ // should we just use all the fallbacks and pick the one with the longest text? - mogery
+ if (isLongEnough || !isGoodStatusCode) {
+ meta.logger.info("Scrape via " + engine + " deemed successful.", {
+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
+ });
+ return engineResult;
+ } else {
+ throw new EngineUnsuccessfulError(engine);
+ }
+}
+
+class WrappedEngineError extends Error {
+ name = "WrappedEngineError";
+ public engine: Engine;
+ public error: any;
+
+ constructor(engine: Engine, error: any) {
+ super("WrappedEngineError");
+ this.engine = engine;
+ this.error = error;
+ }
+}
+
async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
meta.logger.info(`Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`);
@@ -301,168 +338,176 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const fallbackList = buildFallbackList(meta);
+ const snipeAbortController = new AbortController();
+ const snipeAbort: AbortInstance = {
+ signal: snipeAbortController.signal,
+ tier: "engine",
+ throwable() {
+ return new EngineSnipedError();
+ },
+ }
+
+ type EngineBundlePromise = {
+ engine: Engine;
+ unsupportedFeatures: Set<FeatureFlag>;
+ promise: Promise<EngineScrapeResultWithContext>;
+ }
+
+ const remainingEngines = [...fallbackList];
+ let enginePromises: EngineBundlePromise[] = [];
+
+ meta.abort.throwIfAborted();
+
let result: EngineScrapeResultWithContext | null = null;
- const timeToRun =
- meta.options.timeout !== undefined
- ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))
- : (!meta.options.actions && !meta.options.formats.find(x => typeof x === "object" && x.type === "json"))
- ? Math.round(120000 / Math.min(fallbackList.length, 2))
- : undefined;
-
- for (const { engine, unsupportedFeatures } of fallbackList) {
- meta.internalOptions.abort?.throwIfAborted();
- const startedAt = Date.now();
- try {
- meta.logger.info("Scraping via " + engine + "...");
- const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun);
- if (_engineResult.markdown === undefined) {
- // Some engines emit Markdown directly.
- _engineResult.markdown = await parseMarkdown(_engineResult.html);
- }
- const engineResult = _engineResult as EngineScrapeResult & {
- markdown: string;
- };
+ while (remainingEngines.length > 0) {
+ // TODO: REPLACE WITH Engine.maxReasonableTime TODOv2
+ const waitUntilWaterfall = meta.options.timeout !== undefined
+ ? Math.round(meta.options.timeout / Math.min(remainingEngines.length, 2))
+ : (!meta.options.actions && !hasFormatOfType(meta.options.formats, "json"))
+ ? Math.round(120000 / Math.min(remainingEngines.length, 2))
+ : Math.round(300000 / Math.min(remainingEngines.length, 2));
- // Success factors
- const isLongEnough = engineResult.markdown.length > 0;
- const isGoodStatusCode =
- (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||
- engineResult.statusCode === 304;
- const hasNoPageError = engineResult.error === undefined;
- const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);
-
- meta.results[engine] = {
- state: "success",
- result: engineResult,
- factors: { isLongEnough, isGoodStatusCode, hasNoPageError, isLikelyProxyError },
- unsupportedFeatures,
- startedAt,
- finishedAt: Date.now(),
- };
+ const { engine, unsupportedFeatures } = remainingEngines.shift()!;
+
+ if (!isFinite(waitUntilWaterfall) || isNaN(waitUntilWaterfall) || waitUntilWaterfall <= 0) {
+ meta.logger.warn("Invalid waitUntilWaterfall value", {
+ waitUntilWaterfall,
+ timeout: meta.options.timeout,
+ actions: !!meta.options.actions,
+ hasJson: !!meta.options.formats?.find(x => x.type === "json"),
+ remainingEngines: remainingEngines.length,
+ });
+ }
- if (isLikelyProxyError && meta.options.proxy === "auto" && !meta.featureFlags.has("stealthProxy")) {
- meta.logger.info("Scrape via " + engine + " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.");
- throw new AddFeatureError(["stealthProxy"]);
- }
+ meta.logger.info("Scraping via " + engine + "...", {
+ waitUntilWaterfall,
+ });
- // NOTE: TODO: what to do when status code is bad is tough...
- // we cannot just rely on text because error messages can be brief and not hit the limit
- // should we just use all the fallbacks and pick the one with the longest text? - mogery
- if (isLongEnough || !isGoodStatusCode) {
- meta.logger.info("Scrape via " + engine + " deemed successful.", {
- factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
- });
- result = {
- engine,
- unsupportedFeatures,
- result: engineResult as EngineScrapeResult & { markdown: string },
- };
- meta.winnerEngine = engine;
+ enginePromises.push({
+ engine,
+ unsupportedFeatures,
+ promise: (async () => {
+ try {
+ return {
+ engine,
+ unsupportedFeatures,
+ result: await scrapeURLLoopIter(meta, engine, snipeAbort),
+ }
+ } catch (error) {
+ throw new WrappedEngineError(engine, error);
+ }
+ })()
+ });
+
+ while (true) {
+ try {
+ result = await Promise.race([
+ ...enginePromises.map(x => x.promise),
+ new Promise<EngineScrapeResultWithContext>((_, reject) => {
+ setTimeout(() => {
+ reject(new WaterfallNextEngineSignal());
+ }, waitUntilWaterfall);
+ }),
+ new Promise<EngineScrapeResultWithContext>((_, reject) => {
+ setTimeout(() => {
+ try {
+ meta.abort.throwIfAborted();
+ meta.logger.warn("throwIfAborted did not throw");
+ } catch (error) {
+ reject(error);
+ }
+ }, meta.abort.scrapeTimeout() ?? 300000);
+ }),
+ ]);
break;
- }
- } catch (error) {
- if (error instanceof EngineError) {
- meta.logger.warn("Engine " + engine + " could not scrape the page.", {
- error,
- });
- meta.results[engine] = {
- state: "error",
- error: safeguardCircularError(error),
- unexpected: false,
- startedAt,
- finishedAt: Date.now(),
- };
- } else if (error instanceof IndexMissError) {
- meta.logger.info("Engine " + engine + " could not find the page in the index.", {
- error,
- });
- meta.results[engine] = {
- state: "error",
- error: safeguardCircularError(error),
- unexpected: false,
- startedAt,
- finishedAt: Date.now(),
- };
- } else if (error instanceof TimeoutError || (error instanceof Error && error.name === "TimeoutError")) {
- meta.logger.info("Engine " + engine + " timed out while scraping.", {
- error,
- });
- meta.results[engine] = {
- state: "timeout",
- startedAt,
- finishedAt: Date.now(),
- error: safeguardCircularError(error),
- };
- } else if (
- error instanceof AddFeatureError ||
- error instanceof RemoveFeatureError
- ) {
- throw error;
- } else if (error instanceof LLMRefusalError) {
- meta.results[engine] = {
- state: "error",
- error: safeguardCircularError(error),
- unexpected: true,
- startedAt,
- finishedAt: Date.now(),
- };
- error.results = meta.results;
- meta.logger.warn("LLM refusal encountered", { error });
- throw error;
- } else if (error instanceof SiteError) {
- throw error;
- } else if (error instanceof SSLError) {
- throw error;
- } else if (error instanceof DNSResolutionError) {
- throw error;
- } else if (error instanceof ActionError) {
- throw error;
- } else if (error instanceof UnsupportedFileError) {
- throw error;
- } else if (error instanceof PDFAntibotError) {
- throw error;
- } else if (error instanceof TimeoutSignal) {
- throw error;
- } else if (error instanceof PDFInsufficientTimeError) {
- throw error;
- } else if (error instanceof FEPageLoadFailed) {
- meta.results[engine] = {
- state: "error",
- error,
- unexpected: false,
- startedAt,
- finishedAt: Date.now(),
- };
- } else {
- Sentry.captureException(error);
- meta.logger.warn(
- "An unexpected error happened while scraping with " + engine + ".",
- { error },
- );
- meta.results[engine] = {
- state: "error",
- error: safeguardCircularError(error),
- unexpected: true,
- startedAt,
- finishedAt: Date.now(),
- };
+ } catch (error) {
+ if (error instanceof WrappedEngineError) {
+ if (error.error instanceof EngineError) {
+ meta.logger.warn("Engine " + error.engine + " could not scrape the page.", {
+ error: error.error,
+ });
+ } else if (error.error instanceof IndexMissError) {
+ meta.logger.warn("Engine " + error.engine + " could not find the page in the index.", {
+ error: error.error,
+ });
+ } else if (
+ error.error instanceof AddFeatureError ||
+ error.error instanceof RemoveFeatureError ||
+ error.error instanceof SiteError ||
+ error.error instanceof SSLError ||
+ error.error instanceof DNSResolutionError ||
+ error.error instanceof ActionError ||
+ error.error instanceof UnsupportedFileError ||
+ error.error instanceof PDFAntibotError ||
+ error.error instanceof PDFInsufficientTimeError
+ ) {
+ throw error.error;
+ } else if (error.error instanceof LLMRefusalError) {
+ meta.logger.warn("LLM refusal encountered", { error: error.error });
+ throw error.error;
+ } else if (error.error instanceof FEPageLoadFailed) {
+ meta.logger.warn("FEPageLoadFailed encountered!!", { error: error.error });
+ // TODO: what to do about this? TODOv2
+ } else if (error.error instanceof AbortManagerThrownError) {
+ if (error.error.tier === "engine") {
+ meta.logger.warn("Engine " + error.engine + " timed out while scraping.", { error: error.error });
+ } else {
+ throw error.error;
+ }
+ } else {
+ meta.logger.warn("An unexpected error happened while scraping with " + engine + ".", { error });
+ }
+
+ // Filter out the failed engine
+ enginePromises = enginePromises.filter(x => x.engine !== error.engine);
+
+ // If we don't have any engines waterfalled, let's waterfall the next engine
+ if (enginePromises.length === 0) {
+ break;
+ }
+
+ // Otherwise, just keep racing
+ } else if (error instanceof WaterfallNextEngineSignal) {
+ // It's time to waterfall the next engine
+ break;
+ } else if (error instanceof AbortManagerThrownError) {
+ if (error.tier === "engine") {
+ meta.logger.warn("Engine-scoped timeout error received here. Weird!", { error });
+ }
+
+ throw error;
+ } else {
+ meta.logger.warn("Unexpected error while racing engines", { error });
+ throw error;
+ }
}
}
+
+ if (result === null) {
+ meta.logger.info("Waterfalling to next engine...", {
+ waitUntilWaterfall,
+ });
+ } else {
+ break;
+ }
}
+ snipeAbortController.abort();
+
if (result === null) {
- if (meta.results["pdf"]?.state === "timeout") {
- throw meta.results["pdf"].error ?? new TimeoutSignal();
- }
- if (Object.values(meta.results).every(x => x.state === "timeout")) {
- throw new TimeoutSignal();
- } else {
+ // TODO: rectify TODOv2
+ // if (meta.results["pdf"]?.state === "timeout") {
+ // throw meta.results["pdf"].error ?? new TimeoutSignal();
+ // }
+ // if (Object.values(meta.results).every(x => x.state === "timeout")) {
+ // throw new TimeoutSignal();
+ // } else {
throw new NoEnginesLeftError(
fallbackList.map((x) => x.engine),
- meta.results,
+ // meta.results,
);
- }
+ // }
}
let document: Document = {
@@ -478,7 +523,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
numPages: result.result.numPages,
contentType: result.result.contentType,
proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",
- ...((meta.results["index"] || meta.results["index;documents"]) ? (
+ ...(fallbackList.find(x => ["index", "index;documents"].includes(x.engine)) ? (
result.result.cacheInfo ? {
cacheState: "hit",
cachedAt: result.result.cacheInfo.created_at.toISOString(),
@@ -506,8 +551,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
return {
success: true,
document,
- logs: meta.logs,
- engines: meta.results,
};
}
@@ -536,17 +579,14 @@ export async function scrapeURL(
urlToCheck,
options.skipTlsVerification,
meta.logger,
- internalOptions.abort
+ meta.abort.asSignal(),
);
-
if (!isAllowed) {
meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });
return {
success: false,
error: new Error("URL blocked by robots.txt"),
- logs: meta.logs,
- engines: meta.results,
};
}
}
@@ -701,9 +741,10 @@ export async function scrapeURL(
}
}
} catch (error) {
- if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
- throw new FEPageLoadFailed();
- } else if (error instanceof NoEnginesLeftError) {
+ // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {
+ // throw new FEPageLoadFailed();
+ // } else
+ if (error instanceof NoEnginesLeftError) {
meta.logger.warn("scrapeURL: All scraping engines failed!", { error });
} else if (error instanceof LLMRefusalError) {
meta.logger.warn("scrapeURL: LLM refused to extract content", { error });
@@ -730,6 +771,8 @@ export async function scrapeURL(
meta.logger.warn("scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error });
} else if (error instanceof TimeoutSignal) {
throw error;
+ } else if (error instanceof AbortManagerThrownError) {
+ throw error.inner;
} else {
Sentry.captureException(error);
meta.logger.error("scrapeURL: Unexpected error happened", { error });
@@ -739,8 +782,6 @@ export async function scrapeURL(
return {
success: false,
error,
- logs: meta.logs,
- engines: meta.results,
};
}
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/abortManager.ts b/apps/api/src/scraper/scrapeURL/lib/abortManager.ts
new file mode 100644
index 000000000..b9e339da7
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/abortManager.ts
@@ -0,0 +1,90 @@
+export type AbortInstance = {
+ signal: AbortSignal;
+ timesOutAt?: Date;
+ tier: "external" | "scrape" | "engine";
+ throwable: () => any;
+}
+
+export class AbortManager {
+ private aborts: AbortInstance[] = [];
+ private mappedController: AbortController | null = null;
+
+ constructor(...instances: (AbortInstance | undefined | null)[]) {
+ this.aborts = instances.filter(x => x !== undefined && x !== null) as AbortInstance[];
+ }
+
+ add(...instances: (AbortInstance | undefined | null)[]) {
+ const pureInstances = instances.filter(x => x !== undefined && x !== null) as AbortInstance[];
+ this.aborts.push(...pureInstances);
+
+ if (this.mappedController !== null) {
+ for (const abort of pureInstances) {
+ abort.signal.addEventListener("abort", () => {
+ this.mappedController?.abort(new AbortManagerThrownError(abort.tier, abort.throwable()));
+ });
+ }
+ }
+ }
+
+ child(...instances: (AbortInstance | undefined | null)[]): AbortManager {
+ const manager = new AbortManager(...this.aborts, ...instances.filter(x => x !== undefined && x !== null) as AbortInstance[]);
+ return manager;
+ }
+
+ isAborted(): boolean {
+ return this.aborts.some(x => x.signal.aborted);
+ }
+
+ throwIfAborted(): void {
+ for (const abort of this.aborts) {
+ if (abort.signal.aborted) {
+ throw new AbortManagerThrownError(abort.tier, abort.throwable());
+ }
+ }
+ }
+
+ private _mapController() {
+ this.mappedController = new AbortController();
+
+ for (const abort of this.aborts) {
+ abort.signal.addEventListener("abort", () => {
+ this.mappedController?.abort(new AbortManagerThrownError(abort.tier, abort.throwable()));
+ });
+ }
+ }
+
+ asSignal(): AbortSignal {
+ if (this.mappedController === null) {
+ this._mapController();
+ }
+
+ return this.mappedController!.signal;
+ }
+
+ scrapeTimeout(): number | undefined {
+ const timeouts = this.aborts.filter(x => x.tier === "scrape").map(x => x.timesOutAt).filter(x => x !== undefined);
+ if (timeouts.length === 0) {
+ return undefined;
+ }
+ return Math.min(...timeouts.map(x => x.getTime())) - Date.now();
+ }
+
+ engineNearestTimeout(): number | undefined {
+ const timeouts = this.aborts.filter(x => x.tier === "engine").map(x => x.timesOutAt).filter(x => x !== undefined);
+ if (timeouts.length === 0) {
+ return undefined;
+ }
+ return Math.min(...timeouts.map(x => x.getTime())) - Date.now();
+ }
+}
+
+export class AbortManagerThrownError extends Error {
+ name = "AbortManagerThrownError";
+ public tier: AbortInstance["tier"];
+ public inner: any;
+ constructor(tier: AbortInstance["tier"], inner: any) {
+ super("AbortManagerThrownError: " + tier);
+ this.tier = tier;
+ this.inner = inner;
+ }
+}
\ No newline at end of file
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index a9a6188f3..99cb5aa72 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -485,14 +485,6 @@ describe("Standalone scrapeURL tests", () => {
}
};
- // verify that log collection works properly while concurrency is happening
- // expect(out.logs.length).toBeGreaterThan(0);
- const weirdLogs = out.logs.filter((x) => x.scrapeId !== id);
- if (weirdLogs.length > 0) {
- console.warn(JSON.stringify(weirdLogs, replacer));
- }
- expect(weirdLogs.length).toBe(0);
-
if (!out.success) console.error(JSON.stringify(out, replacer));
expect(out.success).toBe(true);
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 98ee0ffde..25ff35eb0 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -6,7 +6,7 @@ import {
TokenUsage,
} from "../../../controllers/v2/types";
import { Logger } from "winston";
-import { EngineResultsTracker, Meta } from "..";
+import { Meta } from "..";
import { logger } from "../../../lib/logger";
import { modelPrices } from "../../../lib/extract/usage/model-prices";
import {
@@ -56,7 +56,6 @@ const getModelLimits = (model: string) => {
export class LLMRefusalError extends Error {
public refusal: string;
- public results: EngineResultsTracker | undefined;
constructor(refusal: string) {
super("LLM refused to extract the website's content");
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index eee58e869..47fd4e0a5 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -316,7 +316,7 @@ export async function addScrapeJobs(
export async function waitForJob(
_job: Job | string,
- timeout: number,
+ timeout: number | null,
logger: Logger = _logger,
): Promise<Document> {
const start = Date.now();
@@ -326,18 +326,18 @@ export async function waitForJob(
logger.debug("Waiting for job to be created");
await new Promise(resolve => setTimeout(resolve, 500));
job = await queue.getJob(_job as string);
- if (Date.now() - start > timeout) {
+ if (Date.now() - start > (timeout ?? 180000)) {
throw new Error("Job wait (concurrency limited)");
}
}
let doc: Document = await Promise.race([
- job.waitUntilFinished(getScrapeQueueEvents(), timeout - (Date.now() - start)),
- new Promise((resolve, reject) => {
+ job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),
+ timeout !== null ? new Promise((resolve, reject) => {
setTimeout(() => {
reject(new Error("Job wait " + (typeof _job == "string" ? "(was concurrency limited)" : "")));
}, Math.max(0, timeout - (Date.now() - start)));
- }),
- ]);
+ }) : null,
+ ].filter(x => x !== null));
logger.debug("Got job");
if (!doc) {