Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2/timeout): initial new waterfalling system (ENG-2922) (#1950)
GergΕ MΓ³ricz9c001a917a8f
diff --git a/apps/api/src/__tests__/snips/v2/map.test.ts b/apps/api/src/__tests__/snips/v2/map.test.tsindex 83e71366d..23c031df7 100644--- a/apps/api/src/__tests__/snips/v2/map.test.ts+++ b/apps/api/src/__tests__/snips/v2/map.test.ts@@ -33,7 +33,7 @@ describe("Map tests", () => {it.concurrent("handles query parameters correctly", async () => {let response = await map({url: "https://www.hfea.gov.uk",- sitemapOnly: true,+ sitemap: "only",useMock: "map-query-params",}, identity);diff --git a/apps/api/src/__tests__/snips/v2/parsers.test.ts b/apps/api/src/__tests__/snips/v2/parsers.test.tsindex f35143ba7..b9257f5cf 100644--- a/apps/api/src/__tests__/snips/v2/parsers.test.ts+++ b/apps/api/src/__tests__/snips/v2/parsers.test.ts@@ -19,7 +19,6 @@ describe("Parsers parameter tests", () => {const response = await scrape({url: pdfUrl,parsers: ["pdf"],- timeout: scrapeTimeout * 2,}, identity);expect(response.markdown).toBeDefined();@@ -31,7 +30,6 @@ describe("Parsers parameter tests", () => {const response = await scrape({url: pdfUrl,parsers: [],- timeout: scrapeTimeout * 2,}, identity);expect(response.markdown).toBeDefined();@@ -42,7 +40,6 @@ describe("Parsers parameter tests", () => {const response = await scrape({url: htmlUrl,parsers: ["pdf"],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -53,7 +50,6 @@ describe("Parsers parameter tests", () => {const response = await scrape({url: htmlUrl,parsers: [],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -66,7 +62,6 @@ describe("Parsers parameter tests", () => {it.concurrent("parses PDF by default when parsers not specified", async () => {const response = await scrape({url: pdfUrl,- timeout: scrapeTimeout * 2,}, identity);expect(response.markdown).toBeDefined();@@ -80,7 +75,6 @@ describe("Parsers parameter tests", () => {const raw = await scrapeRaw({url: pdfUrl,parsers: ["invalid-parser" as any],- timeout: scrapeTimeout,}, identity);expect(raw.statusCode).toBe(400);@@ -92,7 +86,6 @@ describe("Parsers parameter tests", () => {const raw = await scrapeRaw({url: pdfUrl,parsers: "pdf" as any,- timeout: scrapeTimeout,}, identity);expect(raw.statusCode).toBe(400);@@ -104,7 +97,6 @@ describe("Parsers parameter tests", () => {const raw = await scrapeRaw({url: pdfUrl,parsers: { pdf: true } as any,- timeout: scrapeTimeout,}, identity);expect(raw.statusCode).toBe(400);@@ -118,7 +110,6 @@ describe("Parsers parameter tests", () => {const response = await scrape({url: pdfUrl,parsers: ["pdf"],- timeout: scrapeTimeout * 2,}, identity);// Should bill based on number of pages when PDF parsing is enabled@@ -129,7 +120,6 @@ describe("Parsers parameter tests", () => {const response = await scrape({url: pdfUrl,parsers: [],- timeout: scrapeTimeout * 2,}, identity);// Should bill flat rate (1 credit) when PDF parsing is disableddiff --git a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts b/apps/api/src/__tests__/snips/v2/scrape-cache.test.tsindex 2d9deb8bb..b3640d4a6 100644--- a/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-cache.test.ts@@ -20,7 +20,6 @@ describe("V2 Scrape Default maxAge", () => {// First scrape to populate cacheconst data1 = await scrape({url,- timeout: scrapeTimeout,}, identity);expect(data1).toBeDefined();@@ -32,7 +31,6 @@ describe("V2 Scrape Default maxAge", () => {// Second scrape should hit cache with default maxAgeconst data2 = await scrape({url,- timeout: scrapeTimeout,}, identity);expect(data2).toBeDefined();@@ -47,7 +45,6 @@ describe("V2 Scrape Default maxAge", () => {// First scrape to populate cacheconst data1 = await scrape({url,- timeout: scrapeTimeout,}, identity);expect(data1).toBeDefined();@@ -59,7 +56,6 @@ describe("V2 Scrape Default maxAge", () => {// Second scrape with maxAge=0 should miss cacheconst data2 = await scrape({url,- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -74,7 +70,6 @@ describe("V2 Scrape Default maxAge", () => {// First scrape to populate cacheconst data1 = await scrape({url,- timeout: scrapeTimeout,maxAge: 3600000, // 1 hour in milliseconds}, identity);@@ -87,7 +82,6 @@ describe("V2 Scrape Default maxAge", () => {// Second scrape with same maxAge should hit cacheconst data2 = await scrape({url,- timeout: scrapeTimeout,maxAge: 3600000, // 1 hour in milliseconds}, identity);diff --git a/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts b/apps/api/src/__tests__/snips/v2/scrape-formats.test.tsindex 85e4fba9e..d82161eda 100644--- a/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-formats.test.ts@@ -16,7 +16,6 @@ describe("Scrape format variations", () => {const response = await scrape({url: "https://firecrawl.dev",formats: [{ type: "markdown" }],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -28,7 +27,6 @@ describe("Scrape format variations", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["markdown", "html", "links"],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -45,7 +43,6 @@ describe("Scrape format variations", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["screenshot"],- timeout: scrapeTimeout,}, identity);expect(response.screenshot).toBeDefined();@@ -58,7 +55,6 @@ describe("Scrape format variations", () => {const response = await scrape({url: "https://firecrawl.dev",formats: [{ type: "markdown" }],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -74,7 +70,6 @@ describe("Scrape format variations", () => {{ type: "html" },{ type: "links" }],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -96,7 +91,6 @@ describe("Scrape format variations", () => {quality: 80,viewport: { width: 1920, height: 1080 }}],- timeout: scrapeTimeout,}, identity);expect(response.screenshot).toBeDefined();@@ -113,7 +107,6 @@ describe("Scrape format variations", () => {{ type: "html" },"links"],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -134,7 +127,6 @@ describe("Scrape format variations", () => {},{ type: "links" }],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -160,7 +152,6 @@ describe("Scrape format variations", () => {}}}],- timeout: scrapeTimeout,}, identity);expect(response.json).toBeDefined();@@ -180,7 +171,6 @@ describe("Scrape format variations", () => {tag: "test-tag"}],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -193,7 +183,6 @@ describe("Scrape format variations", () => {it.concurrent("default format is markdown when formats not specified", async () => {const response = await scrape({url: "https://firecrawl.dev",- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -204,7 +193,6 @@ describe("Scrape format variations", () => {const raw = await scrapeRaw({url: "https://firecrawl.dev",formats: [{ type: "invalid-format" } as any],- timeout: scrapeTimeout,}, identity);expect(raw.statusCode).toBe(400);@@ -215,7 +203,6 @@ describe("Scrape format variations", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["markdown", "html", "rawHtml", "links"],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -230,7 +217,6 @@ describe("Scrape format variations", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["markdown", "html"],- timeout: scrapeTimeout,}, identity);const keys = Object.keys(response);@@ -243,7 +229,6 @@ describe("Scrape format variations", () => {const response = await scrape({url: "https://firecrawl.dev",formats: [{ type: "markdown" }, { type: "html" }],- timeout: scrapeTimeout,}, identity);const keys = Object.keys(response);@@ -256,19 +241,16 @@ describe("Scrape format variations", () => {const response1 = await scrape({url: "https://firecrawl.dev",formats: ["markdown", "html"],- timeout: scrapeTimeout,}, identity);const response2 = await scrape({url: "https://firecrawl.dev",formats: [{ type: "markdown" }, { type: "html" }],- timeout: scrapeTimeout,}, identity);const response3 = await scrape({url: "https://firecrawl.dev",formats: ["markdown", { type: "html" }],- timeout: scrapeTimeout,}, identity);expect(Object.keys(response1).sort()).toEqual(Object.keys(response2).sort());diff --git a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.tsindex 658e5dcb4..ee202beee 100644--- a/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-skip-tls.test.ts@@ -14,7 +14,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {test("should default skipTlsVerification to true in v2 API", async () => {const data = await scrape({url: "https://expired.badssl.com/",- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -26,7 +25,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {const response = await scrapeRaw({url: "https://expired.badssl.com/",skipTlsVerification: false,- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -41,7 +39,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {test("should work with valid HTTPS sites regardless of skipTlsVerification setting", async () => {const data = await scrape({url: "https://firecrawl.dev",- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -54,7 +51,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {const data = await scrape({url: "https://firecrawl.dev",formats: [{ type: "screenshot", fullPage: false }],- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -67,7 +63,6 @@ describe("V2 Scrape skipTlsVerification Default", () => {const data = await scrape({url: "https://firecrawl.dev",formats: [{ type: "screenshot", fullPage: true }],- timeout: scrapeTimeout,maxAge: 0,}, identity);diff --git a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.tsindex 615dc5b5c..0432db45f 100644--- a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts@@ -25,7 +25,6 @@ describe("V2 Scrape Screenshot Viewport", () => {}}],- timeout: scrapeTimeout,}, identity);expect(data).toBeDefined();@@ -46,7 +45,6 @@ describe("V2 Scrape Screenshot Viewport", () => {}}],- timeout: scrapeTimeout,}, identity);expect(data).toBeDefined();@@ -57,7 +55,6 @@ describe("V2 Scrape Screenshot Viewport", () => {const data = await scrape({url: "https://example.com",formats: ["screenshot"],- timeout: scrapeTimeout,}, identity);expect(data).toBeDefined();@@ -73,7 +70,6 @@ describe("V2 Scrape Screenshot Viewport", () => {fullPage: false}],- timeout: scrapeTimeout,}, identity);expect(data).toBeDefined();@@ -92,7 +88,6 @@ describe("V2 Scrape Screenshot Viewport", () => {}}],- timeout: scrapeTimeout,}, identity);expect(response.status).toBe(400);@@ -111,7 +106,6 @@ describe("V2 Scrape Screenshot Viewport", () => {}}],- timeout: scrapeTimeout,}, identity);expect(response.status).toBe(400);@@ -130,7 +124,6 @@ describe("V2 Scrape Screenshot Viewport", () => {}}],- timeout: scrapeTimeout,}, identity);expect(response.status).toBe(400);diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 27a91b116..55761395b 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -17,7 +17,6 @@ beforeAll(async () => {await scrape({url: changeTrackingTestUrl,formats: ["markdown", "changeTracking"],- timeout: scrapeTimeout,}, identity);}}, 10000 + scrapeTimeout);@@ -26,7 +25,6 @@ describe("Scrape tests", () => {it.concurrent("works", async () => {const response = await scrape({url: "http://firecrawl.dev",- timeout: scrapeTimeout,}, identity);expect(response.markdown).toContain("Firecrawl");@@ -101,14 +99,12 @@ describe("Scrape tests", () => {// it.concurrent("works with Punycode domains", async () => {// await scrape({// url: "http://xn--1lqv92a901a.xn--ses554g/",- // timeout: scrapeTimeout,// }, identity);// }, scrapeTimeout);it.concurrent("handles non-UTF-8 encodings", async () => {const response = await scrape({url: "https://www.rtpro.yamaha.co.jp/RT/docs/misc/kanji-sjis.html",- timeout: scrapeTimeout,}, identity);expect(response.markdown).toContain("γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ γ");@@ -118,7 +114,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["links"],- timeout: scrapeTimeout,}, identity);expect(response.links).toBeDefined();@@ -129,7 +124,6 @@ describe("Scrape tests", () => {it.concurrent("self-hosted proxy works", async () => {const response = await scrape({url: "https://icanhazip.com",- timeout: scrapeTimeout,}, identity);expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);@@ -139,7 +133,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://icanhazip.com",waitFor: 100,- timeout: scrapeTimeout,}, identity);expect(response.markdown?.trim()).toContain(process.env.PROXY_SERVER!.split("://").slice(-1)[0].split(":")[0]);@@ -151,7 +144,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",waitFor: 2000,- timeout: scrapeTimeout,}, identity);expect(response.markdown).toContain("Firecrawl");@@ -163,7 +155,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["rawHtml"],- timeout: scrapeTimeout,}, identity);const obj = JSON.parse(response.rawHtml!);@@ -175,7 +166,6 @@ describe("Scrape tests", () => {it.concurrent("scrape status works", async () => {const response = await scrape({url: "http://firecrawl.dev",- timeout: scrapeTimeout,}, identity);expect(response.markdown).toContain("Firecrawl");@@ -192,7 +182,6 @@ describe("Scrape tests", () => {await scrape({url: "https://firecrawl.dev",blockAds: true,- timeout: scrapeTimeout,}, identity);}, scrapeTimeout);@@ -200,7 +189,6 @@ describe("Scrape tests", () => {await scrape({url: "https://firecrawl.dev",blockAds: false,- timeout: scrapeTimeout,}, identity);}, scrapeTimeout);});@@ -214,7 +202,6 @@ describe("Scrape tests", () => {url,maxAge: scrapeTimeout * 3,storeInCache: false,- timeout: scrapeTimeout,}, identity);expect(response1.metadata.cacheState).toBe("miss");@@ -224,7 +211,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,maxAge: scrapeTimeout * 3,- timeout: scrapeTimeout,}, identity);expect(response2.metadata.cacheState).toBe("miss");@@ -234,7 +220,6 @@ describe("Scrape tests", () => {const response3 = await scrape({url,maxAge: scrapeTimeout * 3,- timeout: scrapeTimeout,}, identity);expect(response3.metadata.cacheState).toBe("hit");@@ -243,7 +228,6 @@ describe("Scrape tests", () => {const response4 = await scrape({url,maxAge: 1,- timeout: scrapeTimeout,}, identity);expect(response4.metadata.cacheState).toBe("miss");@@ -255,7 +239,6 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -265,7 +248,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -279,7 +261,6 @@ describe("Scrape tests", () => {const response1 = await scrape({url,formats: ["screenshot"],- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -290,7 +271,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,formats: ["screenshot"],- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -299,7 +279,6 @@ describe("Scrape tests", () => {const response3 = await scrape({url,formats: [{ type: "screenshot", fullPage: true }],- timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,}, identity);@@ -314,7 +293,6 @@ describe("Scrape tests", () => {const response1 = await scrape({url,formats: [{ type: "screenshot", fullPage: true }],- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -325,7 +303,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,formats: [{ type: "screenshot", fullPage: true }],- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -334,7 +311,6 @@ describe("Scrape tests", () => {const response3 = await scrape({url,formats: ["screenshot"],- timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,}, identity);@@ -349,14 +325,12 @@ describe("Scrape tests", () => {await scrape({url,formats: ["markdown", "changeTracking"],- timeout: scrapeTimeout,maxAge: 0,}, identity);const response1 = await scrape({url,formats: ["markdown", "changeTracking"],- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -367,7 +341,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,formats: ["markdown"],- timeout: scrapeTimeout,maxAge: scrapeTimeout * 3 + indexCooldown,}, identity);@@ -383,7 +356,6 @@ describe("Scrape tests", () => {headers: {"X-Test": "test",},- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -391,7 +363,6 @@ describe("Scrape tests", () => {const response = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2 + indexCooldown,}, identity);@@ -404,7 +375,6 @@ describe("Scrape tests", () => {await scrape({url,- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -412,7 +382,6 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,mobile: true,}, identity);@@ -423,7 +392,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,mobile: true,}, identity);@@ -437,7 +405,6 @@ describe("Scrape tests", () => {const response1 = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout,actions: [{"type": "wait",@@ -451,7 +418,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -464,7 +430,6 @@ describe("Scrape tests", () => {await scrape({url,- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -474,7 +439,6 @@ describe("Scrape tests", () => {url,location: { country: "DE" },maxAge: scrapeTimeout * 2,- timeout: scrapeTimeout,}, identity);expect(response1.metadata.cacheState).toBe("miss");@@ -484,7 +448,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,location: { country: "DE" },- timeout: scrapeTimeout,maxAge: scrapeTimeout * 3,}, identity);@@ -498,7 +461,6 @@ describe("Scrape tests", () => {await scrape({url,blockAds: true,- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -507,7 +469,6 @@ describe("Scrape tests", () => {const response0 = await scrape({url,blockAds: true,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2 + indexCooldown,}, identity);@@ -516,7 +477,6 @@ describe("Scrape tests", () => {const response1 = await scrape({url,blockAds: false,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 3 + indexCooldown,}, identity);@@ -527,7 +487,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,blockAds: false,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 4 + 2 * indexCooldown,}, identity);@@ -541,7 +500,6 @@ describe("Scrape tests", () => {const response1 = await scrape({url,proxy: "stealth",- timeout: scrapeTimeout,maxAge: scrapeTimeout,}, identity);@@ -552,7 +510,6 @@ describe("Scrape tests", () => {const response2 = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2 + indexCooldown,}, identity);@@ -561,7 +518,6 @@ describe("Scrape tests", () => {const response3 = await scrape({url,proxy: "stealth",- timeout: scrapeTimeout,maxAge: scrapeTimeout * 3 + indexCooldown,}, identity);@@ -574,7 +530,6 @@ describe("Scrape tests", () => {await scrape({url,- timeout: scrapeTimeout,maxAge: 0,}, identity);@@ -582,7 +537,6 @@ describe("Scrape tests", () => {const response = await scrape({url,- timeout: scrapeTimeout,maxAge: scrapeTimeout * 2,}, identity);@@ -595,7 +549,6 @@ describe("Scrape tests", () => {const response = await scrape({url: changeTrackingTestUrl,formats: ["markdown", "changeTracking"],- timeout: scrapeTimeout,}, identity);expect(response.changeTracking).toBeDefined();@@ -606,7 +559,6 @@ describe("Scrape tests", () => {const response = await scrape({url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", modes: ["git-diff"] }],- timeout: scrapeTimeout,}, identity);expect(response.changeTracking).toBeDefined();@@ -624,7 +576,6 @@ describe("Scrape tests", () => {const response = await scrape({url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", modes: ["json"], prompt: "Summarize the changes between the previous and current content" }],- timeout: scrapeTimeout,}, identity);expect(response.changeTracking).toBeDefined();@@ -661,7 +612,6 @@ describe("Scrape tests", () => {}}],- timeout: scrapeTimeout,}, identity);expect(response.changeTracking).toBeDefined();@@ -691,7 +641,6 @@ describe("Scrape tests", () => {}}}],- timeout: scrapeTimeout * 2, // takes a while to run, LLM}, identity);expect(response.changeTracking).toBeDefined();@@ -715,13 +664,11 @@ describe("Scrape tests", () => {const response1 = await scrape({url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", tag: uuid1 }],- timeout: scrapeTimeout,}, identity);const response2 = await scrape({url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", tag: uuid2 }],- timeout: scrapeTimeout,}, identity);expect(response1.changeTracking?.previousScrapeAt).toBeNull();@@ -732,7 +679,6 @@ describe("Scrape tests", () => {const response3 = await scrape({url: changeTrackingTestUrl,formats: ["markdown", { type: "changeTracking", tag: uuid1 }],- timeout: scrapeTimeout,}, identity);expect(response3.changeTracking?.previousScrapeAt).not.toBeNull();@@ -744,7 +690,6 @@ describe("Scrape tests", () => {it.concurrent("works without specifying an explicit location", async () => {await scrape({url: "https://iplocation.com",- timeout: scrapeTimeout,}, identity);}, scrapeTimeout);@@ -752,7 +697,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://iplocation.com",location: { country: "US" },- timeout: scrapeTimeout,}, identity);expect(response.markdown).toContain("| Country | United States |");@@ -764,7 +708,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",formats: ["screenshot"],- timeout: scrapeTimeout,}, identity);expect(typeof response.screenshot).toBe("string");@@ -774,7 +717,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "http://firecrawl.dev",formats: [{ type: "screenshot", fullPage: true }],- timeout: scrapeTimeout,}, identity);expect(typeof response.screenshot).toBe("string");@@ -785,7 +727,6 @@ describe("Scrape tests", () => {it.concurrent("works", async () => {const response = await scrape({url: "https://firecrawl.dev",- timeout: scrapeTimeout,actions: [{ type: "pdf" }],}, identity);@@ -800,7 +741,6 @@ describe("Scrape tests", () => {it.concurrent("undefined works", async () => {await scrape({url: "http://firecrawl.dev",- timeout: scrapeTimeout,}, identity);}, scrapeTimeout);@@ -808,7 +748,6 @@ describe("Scrape tests", () => {await scrape({url: "http://firecrawl.dev",proxy: "basic",- timeout: scrapeTimeout,}, identity);}, scrapeTimeout);@@ -816,7 +755,6 @@ describe("Scrape tests", () => {await scrape({url: "http://firecrawl.dev",proxy: "stealth",- timeout: scrapeTimeout * 2,}, identity);}, scrapeTimeout * 2);@@ -824,7 +762,6 @@ describe("Scrape tests", () => {const res = await scrape({url: "http://firecrawl.dev",proxy: "auto",- timeout: scrapeTimeout * 2,}, identity);expect(res.metadata.proxyUsed).toBe("basic");@@ -835,7 +772,6 @@ describe("Scrape tests", () => {// const res = await scrape({// url: "https://eo16f6718vph4un.m.pipedream.net", // always returns 403// proxy: "auto",- // timeout: scrapeTimeout * 2,// }, identity);// expect(res.metadata.proxyUsed).toBe("stealth");@@ -855,17 +791,27 @@ describe("Scrape tests", () => {it.concurrent("blocks long PDFs with insufficient timeout", async () => {const response = await scrapeWithFailure({url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",- timeout: scrapeTimeout,maxAge: 0,+ timeout: 10000,}, identity);expect(response.error).toContain("Insufficient time to process PDF");- }, scrapeTimeout);+ }, 12000);it.concurrent("scrapes long PDFs with sufficient timeout", async () => {const response = await scrape({url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",+ maxAge: 0,timeout: scrapeTimeout * 5,+ }, identity);++ // text on the last page+ expect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");+ }, scrapeTimeout * 5);++ it.concurrent("scrapes long PDFs with default timeout", async () => {+ const response = await scrape({+ url: "https://ecma-international.org/wp-content/uploads/ECMA-262_15th_edition_june_2024.pdf",maxAge: 0,}, identity);@@ -876,7 +822,6 @@ describe("Scrape tests", () => {it.concurrent("scrapes Google Docs links as PDFs", async () => {const response = await scrape({url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",- timeout: scrapeTimeout * 5,}, identity);expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");@@ -885,7 +830,6 @@ describe("Scrape tests", () => {it.concurrent("scrapes Google Slides links as PDFs", async () => {const response = await scrape({url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view",- timeout: scrapeTimeout * 5,maxAge: 0,}, identity);@@ -914,7 +858,6 @@ describe("Scrape tests", () => {},required: ["company_mission", "supports_sso", "is_open_source"],} }],- timeout: scrapeTimeout,}, identity);expect(response).toHaveProperty("json");@@ -934,7 +877,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["summary"],- timeout: scrapeTimeout,}, identity);expect(response.summary).toBeDefined();@@ -946,7 +888,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["markdown", "summary"],- timeout: scrapeTimeout,}, identity);expect(response.summary).toBeDefined();@@ -966,7 +907,6 @@ describe("Scrape tests", () => {}} }}],- timeout: scrapeTimeout,}, identity);expect(response.summary).toBeDefined();@@ -978,7 +918,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://firecrawl.dev",formats: ["markdown", "html", "summary"],- timeout: scrapeTimeout,}, identity);expect(response.summary).toBeDefined();@@ -992,7 +931,6 @@ describe("Scrape tests", () => {it.concurrent("sourceURL stays unnormalized", async () => {const response = await scrape({url: "https://firecrawl.dev/?pagewanted=all&et_blog",- timeout: scrapeTimeout,}, identity);expect(response.metadata.sourceURL).toBe("https://firecrawl.dev/?pagewanted=all&et_blog");@@ -1002,7 +940,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://jsonplaceholder.typicode.com/todos/1",formats: ["markdown"],- timeout: scrapeTimeout,}, identity);expect(response.markdown).toContain("```json");@@ -1013,7 +950,6 @@ describe("Scrape tests", () => {const response = await scrape({url: "https://httpbin.org/html",__experimental_omceDomain: "fake-domain.com",- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();@@ -1025,7 +961,6 @@ describe("Scrape tests", () => {url: "https://httpbin.org/html",__experimental_omceDomain: "test-domain.org",__experimental_omce: true,- timeout: scrapeTimeout,}, identity);expect(response.markdown).toBeDefined();diff --git a/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.ts b/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.tsindex 3acc5ede6..29311a469 100644--- a/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.ts+++ b/apps/api/src/__tests__/snips/v2/system-prompt-rejection.test.ts@@ -27,7 +27,6 @@ describe("V2 System Prompt Rejection", () => {systemPrompt: "Custom system prompt that should be rejected",},],- timeout: scrapeTimeout + 30000,} as any,identity,);@@ -51,7 +50,6 @@ describe("V2 System Prompt Rejection", () => {prompt: "Extract the title",},],- timeout: scrapeTimeout + 30000,},identity,);diff --git a/apps/api/src/controllers/v2/credit-usage.ts b/apps/api/src/controllers/v2/credit-usage.tsnew file mode 100644index 000000000..794861a58--- /dev/null+++ b/apps/api/src/controllers/v2/credit-usage.ts@@ -0,0 +1,46 @@+import { Request, Response } from "express";+import { RequestWithAuth } from "./types";+import { getACUCTeam } from "../auth";+import { logger } from "../../lib/logger";+import { RateLimiterMode } from "../../types";++export async function creditUsageController(+ req: RequestWithAuth,+ res: Response,+): Promise<void> {+ try {+ // If we already have the credit usage info from auth, use it+ if (req.acuc) {+ res.json({+ success: true,+ data: {+ remaining_credits: req.acuc.remaining_credits,+ },+ });+ return;+ }++ // Otherwise fetch fresh data+ const chunk = await getACUCTeam(req.auth.team_id, false, false, RateLimiterMode.Scrape);+ if (!chunk) {+ res.status(404).json({+ success: false,+ error: "Could not find credit usage information",+ });+ return;+ }++ res.json({+ success: true,+ data: {+ remaining_credits: chunk.remaining_credits,+ },+ });+ } catch (error) {+ logger.error("Error in credit usage controller:", error);+ res.status(500).json({+ success: false,+ error: "Internal server error while fetching credit usage",+ });+ }+}diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex eb0941933..87d2577b0 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -324,9 +324,12 @@ export async function mapController(url: req.body.url,search: req.body.search,limit: req.body.limit,- ignoreSitemap: req.body.ignoreSitemap,+ ignoreSitemap: req.body.sitemap === "skip",includeSubdomains: req.body.includeSubdomains,- crawlerOptions: req.body,+ crawlerOptions: {+ ...req.body,+ sitemapOnly: req.body.sitemap === "only",+ },origin: req.body.origin,teamId: req.auth.team_id,abort: abort.signal,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex a9542e55e..df7cf2220 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -97,7 +97,7 @@ export async function scrapeController(let doc: Document;try {- doc = await waitForJob(jobId, timeout + totalWait);+ doc = await waitForJob(jobId, (timeout !== undefined) ? timeout + totalWait : null);} catch (e) {logger.error(`Error in scrapeController`, {startTime,diff --git a/apps/api/src/controllers/v2/token-usage.ts b/apps/api/src/controllers/v2/token-usage.tsnew file mode 100644index 000000000..1e4f6116c--- /dev/null+++ b/apps/api/src/controllers/v2/token-usage.ts@@ -0,0 +1,46 @@+import { Request, Response } from "express";+import { RequestWithAuth } from "./types";+import { getACUC, getACUCTeam } from "../auth";+import { logger } from "../../lib/logger";+import { RateLimiterMode } from "../../types";++export async function tokenUsageController(+ req: RequestWithAuth,+ res: Response,+): Promise<void> {+ try {+ // If we already have the token usage info from auth, use it+ if (req.acuc) {+ res.json({+ success: true,+ data: {+ remaining_tokens: req.acuc.remaining_credits,+ },+ });+ return;+ }++ // Otherwise fetch fresh data+ const chunk = await getACUCTeam(req.auth.team_id, false, false, RateLimiterMode.Extract);+ if (!chunk) {+ res.status(404).json({+ success: false,+ error: "Could not find token usage information",+ });+ return;+ }++ res.json({+ success: true,+ data: {+ remaining_tokens: chunk.remaining_credits,+ },+ });+ } catch (error) {+ logger.error("Error in token usage controller:", error);+ res.status(500).json({+ success: false,+ error: "Internal server error while fetching token usage",+ });+ }+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 76abebcad..ee6c80a30 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -425,7 +425,7 @@ export const extractV1Options = zorigin: z.string().optional().default("api"),integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),urlTrace: z.boolean().default(false),- timeout: z.number().int().positive().finite().safe().default(60000),+ timeout: z.number().int().positive().finite().safe().optional(),__experimental_streamSteps: z.boolean().default(false),__experimental_llmUsage: z.boolean().default(false),__experimental_showSources: z.boolean().default(false),@@ -463,12 +463,10 @@ export type ExtractRequest = z.infer<typeof extractRequestSchema>;export type ExtractRequestInput = z.input<typeof extractRequestSchema>;export const scrapeRequestSchema = baseScrapeOptions- .omit({ timeout: true }).extend({url,origin: z.string().optional().default("api"),integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),- timeout: z.number().int().positive().finite().safe().default(30000),zeroDataRetention: z.boolean().optional(),}).strict(strictMessage)@@ -603,14 +601,14 @@ export type CrawlRequest = z.infer<typeof crawlRequestSchema>;export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;export const mapRequestSchema = crawlerOptions+ .omit({ ignoreSitemap: true }).extend({url,origin: z.string().optional().default("api"),integration: z.nativeEnum(IntegrationEnum).optional().transform(val => val || null),includeSubdomains: z.boolean().default(true),search: z.string().optional(),- ignoreSitemap: z.boolean().default(false),- sitemapOnly: z.boolean().default(false),+ sitemap: z.enum(["only", "include", "skip"]).default("include"),limit: z.number().min(1).max(30000).default(5000),timeout: z.number().positive().finite().optional(),useMock: z.string().optional(),diff --git a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts b/apps/api/src/lib/extract/fire-0/llmExtract-f0.tsindex 6b6af2c60..60a484bf7 100644--- a/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts+++ b/apps/api/src/lib/extract/fire-0/llmExtract-f0.ts@@ -12,7 +12,6 @@ import { generateObject, generateText, LanguageModel } from 'ai';import { jsonSchema } from 'ai';import { getModel } from "../../../lib/generic-ai";import { z } from "zod";-import { EngineResultsTracker, Meta } from "../../../scraper/scrapeURL";// Get max tokens from model pricesconst getModelLimits_F0 = (model: string) => {@@ -34,7 +33,6 @@ const getModelLimits_F0 = (model: string) => {export class LLMRefusalError extends Error {public refusal: string;- public results: EngineResultsTracker | undefined;constructor(refusal: string) {super("LLM refused to extract the website's content");diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex d89a51783..118da61f0 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -10,7 +10,6 @@ import { supabase_service } from "../services/supabase";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";import {- EngineResultsTracker,scrapeURL,ScrapeUrlResponse,} from "../scraper/scrapeURL";@@ -76,7 +75,6 @@ export async function runWebScraper({logger.info("runWebScraper called");let response: ScrapeUrlResponse | undefined = undefined;- let engines: EngineResultsTracker = {};let error: any = undefined;for (let i = 0; i < tries; i++) {@@ -90,7 +88,6 @@ export async function runWebScraper({}response = undefined;- engines = {};error = undefined;try {@@ -116,8 +113,6 @@ export async function runWebScraper({}}- engines = response.engines;-if ((response.document.metadata.statusCode >= 200 &&response.document.metadata.statusCode < 300) ||@@ -128,12 +123,6 @@ export async function runWebScraper({}} catch (_error) {error = _error;- engines =- response !== undefined- ? response.engines- : typeof error === "object" && error !== null- ? ((error as any).results ?? {})- : {};}}@@ -180,8 +169,6 @@ export async function runWebScraper({return {success: false,error,- logs: ["no logs -- error coming from runWebScraper"],- engines,};}}@@ -191,7 +178,6 @@ const saveJob = async (job: Job,result: any,mode: string,- engines?: EngineResultsTracker,) => {try {const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex a9c992ebb..acfdb128c 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -11,6 +11,8 @@ import { mapController } from "../controllers/v2/map";import { crawlErrorsController } from "../controllers/v2/crawl-errors";import { ongoingCrawlsController } from "../controllers/v2/crawl-ongoing";import { scrapeStatusController } from "../controllers/v2/scrape-status";+import { creditUsageController } from "../controllers/v2/credit-usage";+import { tokenUsageController } from "../controllers/v2/token-usage";import {authMiddleware,checkCreditsMiddleware,@@ -96,3 +98,15 @@ v2Router.get(authMiddleware(RateLimiterMode.CrawlStatus),wrap(crawlErrorsController),);++v2Router.get(+ "/credit-usage",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(creditUsageController),+);++v2Router.get(+ "/token-usage",+ authMiddleware(RateLimiterMode.ExtractStatus),+ wrap(tokenUsageController),+);diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 08a0848dc..c4429431e 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -53,7 +53,13 @@ export async function getLinksFromSitemap(...((mode === "fire-engine" && useFireEngine) ? ["fire-engine;tlsclient" as const] : []),],v0DisableJsDom: true,- abort,+ externalAbort: abort ? {+ signal: abort,+ tier: "external",+ throwable() {+ return new Error("Sitemap fetch aborted");+ },+ } : undefined,teamId: "sitemap",zeroDataRetention,},diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.tsindex 11c7304a2..eb27b3bbc 100644--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts@@ -3,10 +3,10 @@ import { EngineScrapeResult } from "..";import { downloadFile } from "../utils/downloadFile";import mammoth from "mammoth";-export async function scrapeDOCX(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {+export async function scrapeDOCX(meta: Meta): Promise<EngineScrapeResult> {const { response, tempFilePath } = await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,- signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeToRun ?? 300000),+ signal: meta.abort.asSignal(),});return {diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex e0e31c8cb..1ab8baca9 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -1,7 +1,7 @@import * as undici from "undici";import { EngineScrapeResult } from "..";import { Meta } from "../..";-import { SSLError, TimeoutError } from "../../error";+import { SSLError } from "../../error";import { specialtyScrapeCheck } from "../utils/specialtyHandler";import {InsecureConnectionError,@@ -13,10 +13,7 @@ import { TextDecoder } from "util";export async function scrapeURLWithFetch(meta: Meta,- timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const timeout = timeToRun ?? 300000;-const mockOptions = {url: meta.rewrittenUrl ?? meta.url,@@ -55,23 +52,12 @@ export async function scrapeURLWithFetch(};} else {try {- const x = await Promise.race([- undici.fetch(meta.rewrittenUrl ?? meta.url, {- dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,- redirect: "follow",- headers: meta.options.headers,- signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),- }),- (async () => {- await new Promise((resolve) =>- setTimeout(() => resolve(null), timeout),- );- throw new TimeoutError(- "Fetch was unable to scrape the page before timing out",- { cause: { timeout } },- );- })(),- ]);+ const x = await undici.fetch(meta.rewrittenUrl ?? meta.url, {+ dispatcher: meta.options.skipTlsVerification ? secureDispatcherSkipTlsVerification : secureDispatcher,+ redirect: "follow",+ headers: meta.options.headers,+ signal: meta.abort.asSignal(),+ });const buf = Buffer.from(await x.arrayBuffer());let text = buf.toString("utf8");diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 5fd30ff38..71ec86c24 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -19,7 +19,6 @@ import {DNSResolutionError,SiteError,SSLError,- TimeoutError,UnsupportedFileError,FEPageLoadFailed,} from "../../error";@@ -28,8 +27,9 @@ import { specialtyScrapeCheck } from "../utils/specialtyHandler";import { fireEngineDelete } from "./delete";import { MockState } from "../../lib/mock";import { getInnerJSON } from "../../../../lib/html-transformer";-import { hasFormatOfType, hasAnyFormatOfTypes } from "../../../../lib/format-utils";-import { Action, TimeoutSignal } from "../../../../controllers/v1/types";+import { hasFormatOfType } from "../../../../lib/format-utils";+import { Action } from "../../../../controllers/v1/types";+import { AbortManagerThrownError } from "../../lib/abortManager";// This function does not take `Meta` on purpose. It may not access any// meta values to construct the request -- that must be done by the@@ -43,7 +43,6 @@ async function performFireEngineScrape<meta: Meta,logger: Logger,request: FireEngineScrapeRequestCommon & Engine,- timeout: number,mock: MockState | null,abort?: AbortSignal,production = true,@@ -56,13 +55,11 @@ async function performFireEngineScrape<production,);- const startTime = Date.now();const errorLimit = 3;let errors: any[] = [];let status: FireEngineCheckStatusSuccess | undefined = undefined;while (status === undefined) {- abort?.throwIfAborted();if (errors.length >= errorLimit) {logger.error("Error limit hit.", { errors });fireEngineDelete(@@ -80,16 +77,7 @@ async function performFireEngineScrape<});}- if (Date.now() - startTime > timeout) {- logger.info(- "Fire-engine was unable to scrape the page before timing out.",- { errors, timeout },- );- throw new TimeoutError(- "Fire-engine was unable to scrape the page before timing out",- { cause: { errors, timeout } },- );- }+ meta.abort.throwIfAborted();try {status = await fireEngineCheckStatus(@@ -127,7 +115,7 @@ async function performFireEngineScrape<jobId: scrape.jobId,});throw error;- } else if (error instanceof TimeoutSignal) {+ } else if (error instanceof AbortManagerThrownError) {fireEngineDelete(logger.child({method: "performFireEngineScrape/fireEngineDelete",@@ -138,7 +126,7 @@ async function performFireEngineScrape<undefined,production,);- throw new TimeoutError("Fire-engine timed out while checking status", { cause: { errors, timeout } });+ throw error;} else {errors.push(error);logger.debug(@@ -189,7 +177,6 @@ async function performFireEngineScrape<export async function scrapeURLWithFireEngineChromeCDP(meta: Meta,- timeToRun: number | undefined,): Promise<EngineScrapeResult> {const actions: Action[] = [// Transform waitFor option into an action (unsupported by chrome-cdp)@@ -223,8 +210,6 @@ export async function scrapeURLWithFireEngineChromeCDP(0,);- const timeout = (timeToRun ?? 300000) + totalWait;-const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {url: meta.rewrittenUrl ?? meta.url,@@ -240,7 +225,7 @@ export async function scrapeURLWithFireEngineChromeCDP(priority: meta.internalOptions.priority,geolocation: meta.options.location,mobile: meta.options.mobile,- timeout, // TODO: better timeout logic+ timeout: meta.abort.scrapeTimeout() ?? 300000,disableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,mobileProxy: meta.featureFlags.has("stealthProxy"),saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS,@@ -254,9 +239,8 @@ export async function scrapeURLWithFireEngineChromeCDP(request,}),request,- timeout,meta.mock,- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),+ meta.abort.asSignal(),true,);@@ -311,10 +295,8 @@ export async function scrapeURLWithFireEngineChromeCDP(export async function scrapeURLWithFireEnginePlaywright(meta: Meta,- timeToRun: number | undefined,): Promise<EngineScrapeResult> {const totalWait = meta.options.waitFor;- const timeout = (timeToRun ?? 300000) + totalWait;const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestPlaywright = {@@ -331,7 +313,7 @@ export async function scrapeURLWithFireEnginePlaywright(blockAds: meta.options.blockAds,mobileProxy: meta.featureFlags.has("stealthProxy"),- timeout,+ timeout: meta.abort.scrapeTimeout() ?? 300000,saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,};@@ -343,9 +325,8 @@ export async function scrapeURLWithFireEnginePlaywright(request,}),request,- timeout,meta.mock,- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),+ meta.abort.asSignal(),);if (!response.url) {@@ -378,10 +359,7 @@ export async function scrapeURLWithFireEnginePlaywright(export async function scrapeURLWithFireEngineTLSClient(meta: Meta,- timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const timeout = timeToRun ?? 30000;-const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestTLSClient = {url: meta.rewrittenUrl ?? meta.url,@@ -396,7 +374,7 @@ export async function scrapeURLWithFireEngineTLSClient(disableJsDom: meta.internalOptions.v0DisableJsDom,mobileProxy: meta.featureFlags.has("stealthProxy"),- timeout,+ timeout: meta.abort.scrapeTimeout() ?? 300000,saveScrapeResultToGCS: !meta.internalOptions.zeroDataRetention && meta.internalOptions.saveScrapeResultToGCS,zeroDataRetention: meta.internalOptions.zeroDataRetention,};@@ -408,9 +386,8 @@ export async function scrapeURLWithFireEngineTLSClient(request,}),request,- timeout,meta.mock,- meta.internalOptions.abort ?? AbortSignal.timeout(timeout),+ meta.abort.asSignal(),);if (!response.url) {diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex bc734e010..9e4d380a8 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -127,7 +127,6 @@ export type EngineScrapeResult = {const engineHandlers: {[E in Engine]: (meta: Meta,- timeToRun: number | undefined,) => Promise<EngineScrapeResult>;} = {index: scrapeURLWithIndex,@@ -513,7 +512,6 @@ export function buildFallbackList(meta: Meta): {export async function scrapeURLWithEngine(meta: Meta,engine: Engine,- timeToRun: number | undefined,): Promise<EngineScrapeResult> {const fn = engineHandlers[engine];const logger = meta.logger.child({@@ -525,5 +523,5 @@ export async function scrapeURLWithEngine(logger,};- return await fn(_meta, timeToRun);+ return await fn(_meta);}diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex 789e0b637..769350b26 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -2,7 +2,7 @@ import { Document } from "../../../../controllers/v1/types";import { EngineScrapeResult } from "..";import { Meta } from "../..";import { getIndexFromGCS, hashURL, index_supabase_service, normalizeURLForIndex, saveIndexToGCS, generateURLSplits, addIndexInsertJob, generateDomainSplits, addOMCEJob, addDomainFrequencyJob } from "../../../../services";-import { EngineError, IndexMissError, TimeoutError } from "../../error";+import { EngineError, IndexMissError } from "../../error";import crypto from "crypto";export async function sendDocumentToIndex(meta: Meta, document: Document) {@@ -136,7 +136,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {const errorCountToRegister = 3;-export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {+export async function scrapeURLWithIndex(meta: Meta): Promise<EngineScrapeResult> {const normalizedURL = normalizeURLForIndex(meta.url);const urlHash = hashURL(normalizedURL);@@ -165,14 +165,9 @@ export async function scrapeURLWithIndex(meta: Meta, timeToRun: number | undefinselector = selector.is("location_languages", null);}- const { data, error } = await Promise.race([- selector- .order("created_at", { ascending: false })- .limit(5),- new Promise<{ data: { id: any; created_at: any; status: any }[], error: any }>((resolve, reject) => {- setTimeout(() => reject(new TimeoutError()), timeToRun ?? 10000);- }),- ]);+ const { data, error } = await selector+ .order("created_at", { ascending: false })+ .limit(5);if (error || !data) {throw new EngineError("Failed to retrieve URL from DB index", {diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 177104b3c..f63ddaaeb 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -12,13 +12,13 @@ import {PDFInsufficientTimeError,PDFPrefetchFailed,RemoveFeatureError,- TimeoutError,} from "../../error";import { readFile, unlink } from "node:fs/promises";import path from "node:path";import type { Response } from "undici";import { getPageCount } from "../../../../lib/pdf-parser";import { getPdfResultFromCache, savePdfResultToCache } from "../../../../lib/gcs-pdf-cache";+import { AbortManagerThrownError } from "../../lib/abortManager";type PDFProcessorResult = { html: string; markdown?: string };@@ -28,15 +28,12 @@ const MILLISECONDS_PER_PAGE = 150;async function scrapePDFWithRunPodMU(meta: Meta,tempFilePath: string,- timeToRun: number | undefined,base64Content: string,): Promise<PDFProcessorResult> {meta.logger.debug("Processing PDF document with RunPod MU", {tempFilePath,});- const preCacheCheckStartTime = Date.now();-try {const cachedResult = await getPdfResultFromCache(base64Content);if (cachedResult) {@@ -52,14 +49,7 @@ async function scrapePDFWithRunPodMU(});}- const timeout = timeToRun- ? timeToRun - (Date.now() - preCacheCheckStartTime)- : undefined;- if (timeout && timeout < 0) {- throw new TimeoutError("MU PDF parser already timed out before call");- }-- const abort = timeout ? AbortSignal.timeout(timeout) : undefined;+ meta.abort.throwIfAborted();const podStart = await robustFetch({url:@@ -72,7 +62,7 @@ async function scrapePDFWithRunPodMU(input: {file_content: base64Content,filename: path.basename(tempFilePath) + ".pdf",- timeout,+ timeout: meta.abort.scrapeTimeout(),created_at: Date.now(),},},@@ -89,19 +79,17 @@ async function scrapePDFWithRunPodMU(.optional(),}),mock: meta.mock,- abort,+ abort: meta.abort.asSignal(),});--let status: string = podStart.status;let result: { markdown: string } | undefined = podStart.output;if (status === "IN_QUEUE" || status === "IN_PROGRESS") {do {- abort?.throwIfAborted();+ meta.abort.throwIfAborted();await new Promise((resolve) => setTimeout(resolve, 2500));- abort?.throwIfAborted();+ meta.abort.throwIfAborted();const podStatus = await robustFetch({url: `https://api.runpod.ai/v2/${process.env.RUNPOD_MU_POD_ID}/status/${podStart.id}`,method: "GET",@@ -120,7 +108,7 @@ async function scrapePDFWithRunPodMU(.optional(),}),mock: meta.mock,- abort,+ abort: meta.abort.asSignal(),});status = podStatus.status;result = podStatus.output;@@ -171,10 +159,7 @@ async function scrapePDFWithParsePDF(export async function scrapePDF(meta: Meta,- timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const startTime = Date.now();-const shouldParsePDF = meta.options.parsers?.includes("pdf") ?? true;if (!shouldParsePDF) {@@ -241,7 +226,7 @@ export async function scrapePDF(}const pageCount = await getPageCount(tempFilePath);- if (pageCount * MILLISECONDS_PER_PAGE > (timeToRun ?? Infinity)) {+ if (pageCount * MILLISECONDS_PER_PAGE > (meta.abort.scrapeTimeout() ?? Infinity)) {throw new PDFInsufficientTimeError(pageCount,pageCount * MILLISECONDS_PER_PAGE + 5000,@@ -252,8 +237,6 @@ export async function scrapePDF(const base64Content = (await readFile(tempFilePath)).toString("base64");- const remainingTime = timeToRun ? timeToRun - (Date.now() - startTime) : undefined;-// First try RunPod MU if conditions are metif (base64Content.length < MAX_FILE_SIZE &&@@ -269,28 +252,27 @@ export async function scrapePDF(}),},tempFilePath,- remainingTime,base64Content,);} catch (error) {if (error instanceof RemoveFeatureError ||- error instanceof TimeoutError+ error instanceof AbortManagerThrownError) {throw error;- } else if (- (error instanceof Error && error.name === "TimeoutError") ||- (error instanceof Error &&- error.message === "Request failed" &&- error.cause &&- error.cause instanceof Error &&- error.cause.name === "TimeoutError") ||- (error instanceof Error && error.name === "TimeoutSignal")- ) {- meta.logger.warn("RunPod MU timed out", { remainingTime });- throw new TimeoutError(- "PDF parsing timed out, please increase the timeout parameter in your scrape request",- );+ // } else if (+ // (error instanceof Error && error.name === "TimeoutError") ||+ // (error instanceof Error &&+ // error.message === "Request failed" &&+ // error.cause &&+ // error.cause instanceof Error &&+ // error.cause.name === "TimeoutError") ||+ // (error instanceof Error && error.name === "TimeoutSignal")+ // ) {+ // meta.logger.warn("RunPod MU timed out", { timeout: remainingTime });+ // throw new TimeoutError(+ // "PDF parsing timed out, please increase the timeout parameter in your scrape request",+ // );}meta.logger.warn("RunPod MU failed to parse PDF (could be due to timeout) -- falling back to parse-pdf",diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.tsindex 98285543b..dd0fc29fc 100644--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts@@ -1,48 +1,35 @@import { z } from "zod";import { EngineScrapeResult } from "..";import { Meta } from "../..";-import { TimeoutError } from "../../error";import { robustFetch } from "../../lib/fetch";import { getInnerJSON } from "../../../../lib/html-transformer";export async function scrapeURLWithPlaywright(meta: Meta,- timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const timeout = (timeToRun ?? 300000) + meta.options.waitFor;-- const response = await Promise.race([- await robustFetch({- url: process.env.PLAYWRIGHT_MICROSERVICE_URL!,- headers: {- "Content-Type": "application/json",- },- body: {- url: meta.rewrittenUrl ?? meta.url,- wait_after_load: meta.options.waitFor,- timeout,- headers: meta.options.headers,- skip_tls_verification: meta.options.skipTlsVerification,- },- method: "POST",- logger: meta.logger.child("scrapeURLWithPlaywright/robustFetch"),- schema: z.object({- content: z.string(),- pageStatusCode: z.number(),- pageError: z.string().optional(),- contentType: z.string().optional(),- }),- mock: meta.mock,- abort: AbortSignal.timeout(timeout),+ const response = await robustFetch({+ url: process.env.PLAYWRIGHT_MICROSERVICE_URL!,+ headers: {+ "Content-Type": "application/json",+ },+ body: {+ url: meta.rewrittenUrl ?? meta.url,+ wait_after_load: meta.options.waitFor,+ timeout: meta.abort.scrapeTimeout(),+ headers: meta.options.headers,+ skip_tls_verification: meta.options.skipTlsVerification,+ },+ method: "POST",+ logger: meta.logger.child("scrapeURLWithPlaywright/robustFetch"),+ schema: z.object({+ content: z.string(),+ pageStatusCode: z.number(),+ pageError: z.string().optional(),+ contentType: z.string().optional(),}),- (async () => {- await new Promise((resolve) => setTimeout(() => resolve(null), timeout));- throw new TimeoutError(- "Playwright was unable to scrape the page before timing out",- { cause: { timeout } },- );- })(),- ]);+ mock: meta.mock,+ abort: meta.abort.asSignal(),+ });if (response.contentType?.includes("application/json")) {response.content = await getInnerJSON(response.content);diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex b2ac60aaa..295b208b0 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -1,4 +1,4 @@-import { EngineResultsTracker, Meta } from ".";+import { Meta } from ".";import { Engine, FeatureFlag } from "./engines";export class EngineError extends Error {@@ -7,22 +7,14 @@ export class EngineError extends Error {}}-export class TimeoutError extends Error {- constructor(message?: string, options?: ErrorOptions) {- super(message, options);- }-}-export class NoEnginesLeftError extends Error {public fallbackList: Engine[];- public results: EngineResultsTracker;- constructor(fallbackList: Engine[], results: EngineResultsTracker) {+ constructor(fallbackList: Engine[]) {super("All scraping engines failed! -- Double check the URL to make sure it's not broken. If the issue persists, contact us at [email protected].",);this.fallbackList = fallbackList;- this.results = results;}}@@ -127,3 +119,41 @@ export class FEPageLoadFailed extends Error {super("The page failed to load with the specified timeout. Please increase the timeout parameter in your request.");}}++export class ScrapeURLScopedTimeoutError extends Error {+ constructor(message: string, options?: ErrorOptions) {+ super(message, options);+ }+}++export class ScrapeTimeoutError extends ScrapeURLScopedTimeoutError {+ name = "ScrapeTimeoutError";++ constructor() {+ super("Scrape timed out");+ }+}++export class EngineSnipedError extends ScrapeURLScopedTimeoutError {+ name = "EngineSnipedError";++ constructor() {+ super("Engine got sniped");+ }+}++export class EngineUnsuccessfulError extends Error {+ name = "EngineUnsuccessfulError";++ constructor(engine: Engine) {+ super(`Engine ${engine} was unsuccessful`);+ }+}++export class WaterfallNextEngineSignal extends Error {+ name = "WaterfallNextEngineSignal";++ constructor() {+ super("Waterfall next engine");+ }+}\ No newline at end of filediff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 25088db8c..bb22a2875 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -21,7 +21,6 @@ import {PDFAntibotError,RemoveFeatureError,SiteError,- TimeoutError,UnsupportedFileError,SSLError,PDFInsufficientTimeError,@@ -30,6 +29,10 @@ import {ZDRViolationError,PDFPrefetchFailed,FEPageLoadFailed,+ ScrapeTimeoutError,+ EngineSnipedError,+ WaterfallNextEngineSignal,+ EngineUnsuccessfulError,} from "./error";import { executeTransformers } from "./transformers";import { LLMRefusalError } from "./transformers/llmExtract";@@ -39,6 +42,7 @@ import { CostTracking } from "../../lib/extract/extraction-service";import { robustFetch } from "./lib/fetch";import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";import { checkRobotsTxt } from "../../lib/robots-txt";+import { AbortInstance, AbortManager, AbortManagerThrownError } from "./lib/abortManager";export type ScrapeUrlResponse = (| {@@ -49,10 +53,7 @@ export type ScrapeUrlResponse = (success: false;error: any;}-) & {- logs: any[];- engines: EngineResultsTracker;-};+);export type Meta = {id: string;@@ -61,7 +62,7 @@ export type Meta = {options: ScrapeOptions;internalOptions: InternalOptions;logger: Logger;- logs: any[];+ abort: AbortManager;featureFlags: Set<FeatureFlag>;mock: MockState | null;pdfPrefetch: {@@ -72,7 +73,6 @@ export type Meta = {} | null | undefined; // undefined: no prefetch yet, null: prefetch came back emptycostTracking: CostTracking;winnerEngine?: Engine;- results: EngineResultsTracker;};function buildFeatureFlags(@@ -188,7 +188,6 @@ async function buildMetaObject(team_id: internalOptions.teamId,crawlId: internalOptions.crawlId,});- const logs: any[] = [];return {id,@@ -197,7 +196,17 @@ async function buildMetaObject(options,internalOptions,logger,- logs,+ abort: new AbortManager(+ internalOptions.externalAbort,+ options.timeout !== undefined ? {+ signal: AbortSignal.timeout(options.timeout),+ tier: "scrape",+ timesOutAt: new Date(Date.now() + options.timeout),+ throwable() {+ return new ScrapeTimeoutError();+ },+ } : undefined,+ ),featureFlags: buildFeatureFlags(url, options, internalOptions),mock:options.useMock !== undefined@@ -205,7 +214,6 @@ async function buildMetaObject(: null,pdfPrefetch: undefined,costTracking,- results: {},};}@@ -221,7 +229,7 @@ export type InternalOptions = {v0DisableJsDom?: boolean;disableSmartWaitCache?: boolean; // Passed along to fire-engineisBackgroundIndex?: boolean;- abort?: AbortSignal;+ externalAbort?: AbortInstance;urlInvisibleInCurrentCrawl?: boolean;unnormalizedSourceURL?: string;@@ -236,29 +244,6 @@ export type InternalOptions = {v1OriginalFormat?: "extract" | "json"; // Track original v1 format for backward compatibility};-export type EngineResultsTracker = {- [E in Engine]?: (- | {- state: "error";- error: any;- unexpected: boolean;- }- | {- state: "success";- result: EngineScrapeResult & { markdown: string };- factors: Record<string, boolean>;- unsupportedFeatures: Set<FeatureFlag>;- }- | {- state: "timeout";- error: any;- }- ) & {- startedAt: number;- finishedAt: number;- };-};-export type EngineScrapeResultWithContext = {engine: Engine;unsupportedFeatures: Set<FeatureFlag>;@@ -275,6 +260,58 @@ function safeguardCircularError<T>(error: T): T {}}+async function scrapeURLLoopIter(meta: Meta, engine: Engine, snipeAbort): Promise<EngineScrapeResult & { markdown: string }> {+ const _engineResult = await scrapeURLWithEngine({+ ...meta,+ abort: meta.abort.child(snipeAbort),+ }, engine);++ if (_engineResult.markdown === undefined) {+ _engineResult.markdown = await parseMarkdown(_engineResult.html);+ }++ const engineResult = _engineResult as EngineScrapeResult & {+ markdown: string;+ };++ // Success factors+ const isLongEnough = engineResult.markdown.length > 0;+ const isGoodStatusCode =+ (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||+ engineResult.statusCode === 304;+ const hasNoPageError = engineResult.error === undefined;+ const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);++ if (isLikelyProxyError && meta.options.proxy === "auto" && !meta.featureFlags.has("stealthProxy")) {+ meta.logger.info("Scrape via " + engine + " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.");+ throw new AddFeatureError(["stealthProxy"]);+ }++ // NOTE: TODO: what to do when status code is bad is tough...+ // we cannot just rely on text because error messages can be brief and not hit the limit+ // should we just use all the fallbacks and pick the one with the longest text? - mogery+ if (isLongEnough || !isGoodStatusCode) {+ meta.logger.info("Scrape via " + engine + " deemed successful.", {+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },+ });+ return engineResult;+ } else {+ throw new EngineUnsuccessfulError(engine);+ }+}++class WrappedEngineError extends Error {+ name = "WrappedEngineError";+ public engine: Engine;+ public error: any;++ constructor(engine: Engine, error: any) {+ super("WrappedEngineError");+ this.engine = engine;+ this.error = error;+ }+}+async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {meta.logger.info(`Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`);@@ -301,168 +338,176 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const fallbackList = buildFallbackList(meta);+ const snipeAbortController = new AbortController();+ const snipeAbort: AbortInstance = {+ signal: snipeAbortController.signal,+ tier: "engine",+ throwable() {+ return new EngineSnipedError();+ },+ }++ type EngineBundlePromise = {+ engine: Engine;+ unsupportedFeatures: Set<FeatureFlag>;+ promise: Promise<EngineScrapeResultWithContext>;+ }++ const remainingEngines = [...fallbackList];+ let enginePromises: EngineBundlePromise[] = [];++ meta.abort.throwIfAborted();+let result: EngineScrapeResultWithContext | null = null;- const timeToRun =- meta.options.timeout !== undefined- ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))- : (!meta.options.actions && !meta.options.formats.find(x => typeof x === "object" && x.type === "json"))- ? Math.round(120000 / Math.min(fallbackList.length, 2))- : undefined;-- for (const { engine, unsupportedFeatures } of fallbackList) {- meta.internalOptions.abort?.throwIfAborted();- const startedAt = Date.now();- try {- meta.logger.info("Scraping via " + engine + "...");- const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun);- if (_engineResult.markdown === undefined) {- // Some engines emit Markdown directly.- _engineResult.markdown = await parseMarkdown(_engineResult.html);- }- const engineResult = _engineResult as EngineScrapeResult & {- markdown: string;- };+ while (remainingEngines.length > 0) {+ // TODO: REPLACE WITH Engine.maxReasonableTime TODOv2+ const waitUntilWaterfall = meta.options.timeout !== undefined+ ? Math.round(meta.options.timeout / Math.min(remainingEngines.length, 2))+ : (!meta.options.actions && !hasFormatOfType(meta.options.formats, "json"))+ ? Math.round(120000 / Math.min(remainingEngines.length, 2))+ : Math.round(300000 / Math.min(remainingEngines.length, 2));- // Success factors- const isLongEnough = engineResult.markdown.length > 0;- const isGoodStatusCode =- (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||- engineResult.statusCode === 304;- const hasNoPageError = engineResult.error === undefined;- const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);-- meta.results[engine] = {- state: "success",- result: engineResult,- factors: { isLongEnough, isGoodStatusCode, hasNoPageError, isLikelyProxyError },- unsupportedFeatures,- startedAt,- finishedAt: Date.now(),- };+ const { engine, unsupportedFeatures } = remainingEngines.shift()!;++ if (!isFinite(waitUntilWaterfall) || isNaN(waitUntilWaterfall) || waitUntilWaterfall <= 0) {+ meta.logger.warn("Invalid waitUntilWaterfall value", {+ waitUntilWaterfall,+ timeout: meta.options.timeout,+ actions: !!meta.options.actions,+ hasJson: !!meta.options.formats?.find(x => x.type === "json"),+ remainingEngines: remainingEngines.length,+ });+ }- if (isLikelyProxyError && meta.options.proxy === "auto" && !meta.featureFlags.has("stealthProxy")) {- meta.logger.info("Scrape via " + engine + " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.");- throw new AddFeatureError(["stealthProxy"]);- }+ meta.logger.info("Scraping via " + engine + "...", {+ waitUntilWaterfall,+ });- // NOTE: TODO: what to do when status code is bad is tough...- // we cannot just rely on text because error messages can be brief and not hit the limit- // should we just use all the fallbacks and pick the one with the longest text? - mogery- if (isLongEnough || !isGoodStatusCode) {- meta.logger.info("Scrape via " + engine + " deemed successful.", {- factors: { isLongEnough, isGoodStatusCode, hasNoPageError },- });- result = {- engine,- unsupportedFeatures,- result: engineResult as EngineScrapeResult & { markdown: string },- };- meta.winnerEngine = engine;+ enginePromises.push({+ engine,+ unsupportedFeatures,+ promise: (async () => {+ try {+ return {+ engine,+ unsupportedFeatures,+ result: await scrapeURLLoopIter(meta, engine, snipeAbort),+ }+ } catch (error) {+ throw new WrappedEngineError(engine, error);+ }+ })()+ });++ while (true) {+ try {+ result = await Promise.race([+ ...enginePromises.map(x => x.promise),+ new Promise<EngineScrapeResultWithContext>((_, reject) => {+ setTimeout(() => {+ reject(new WaterfallNextEngineSignal());+ }, waitUntilWaterfall);+ }),+ new Promise<EngineScrapeResultWithContext>((_, reject) => {+ setTimeout(() => {+ try {+ meta.abort.throwIfAborted();+ meta.logger.warn("throwIfAborted did not throw");+ } catch (error) {+ reject(error);+ }+ }, meta.abort.scrapeTimeout() ?? 300000);+ }),+ ]);break;- }- } catch (error) {- if (error instanceof EngineError) {- meta.logger.warn("Engine " + engine + " could not scrape the page.", {- error,- });- meta.results[engine] = {- state: "error",- error: safeguardCircularError(error),- unexpected: false,- startedAt,- finishedAt: Date.now(),- };- } else if (error instanceof IndexMissError) {- meta.logger.info("Engine " + engine + " could not find the page in the index.", {- error,- });- meta.results[engine] = {- state: "error",- error: safeguardCircularError(error),- unexpected: false,- startedAt,- finishedAt: Date.now(),- };- } else if (error instanceof TimeoutError || (error instanceof Error && error.name === "TimeoutError")) {- meta.logger.info("Engine " + engine + " timed out while scraping.", {- error,- });- meta.results[engine] = {- state: "timeout",- startedAt,- finishedAt: Date.now(),- error: safeguardCircularError(error),- };- } else if (- error instanceof AddFeatureError ||- error instanceof RemoveFeatureError- ) {- throw error;- } else if (error instanceof LLMRefusalError) {- meta.results[engine] = {- state: "error",- error: safeguardCircularError(error),- unexpected: true,- startedAt,- finishedAt: Date.now(),- };- error.results = meta.results;- meta.logger.warn("LLM refusal encountered", { error });- throw error;- } else if (error instanceof SiteError) {- throw error;- } else if (error instanceof SSLError) {- throw error;- } else if (error instanceof DNSResolutionError) {- throw error;- } else if (error instanceof ActionError) {- throw error;- } else if (error instanceof UnsupportedFileError) {- throw error;- } else if (error instanceof PDFAntibotError) {- throw error;- } else if (error instanceof TimeoutSignal) {- throw error;- } else if (error instanceof PDFInsufficientTimeError) {- throw error;- } else if (error instanceof FEPageLoadFailed) {- meta.results[engine] = {- state: "error",- error,- unexpected: false,- startedAt,- finishedAt: Date.now(),- };- } else {- Sentry.captureException(error);- meta.logger.warn(- "An unexpected error happened while scraping with " + engine + ".",- { error },- );- meta.results[engine] = {- state: "error",- error: safeguardCircularError(error),- unexpected: true,- startedAt,- finishedAt: Date.now(),- };+ } catch (error) {+ if (error instanceof WrappedEngineError) {+ if (error.error instanceof EngineError) {+ meta.logger.warn("Engine " + error.engine + " could not scrape the page.", {+ error: error.error,+ });+ } else if (error.error instanceof IndexMissError) {+ meta.logger.warn("Engine " + error.engine + " could not find the page in the index.", {+ error: error.error,+ });+ } else if (+ error.error instanceof AddFeatureError ||+ error.error instanceof RemoveFeatureError ||+ error.error instanceof SiteError ||+ error.error instanceof SSLError ||+ error.error instanceof DNSResolutionError ||+ error.error instanceof ActionError ||+ error.error instanceof UnsupportedFileError ||+ error.error instanceof PDFAntibotError ||+ error.error instanceof PDFInsufficientTimeError+ ) {+ throw error.error;+ } else if (error.error instanceof LLMRefusalError) {+ meta.logger.warn("LLM refusal encountered", { error: error.error });+ throw error.error;+ } else if (error.error instanceof FEPageLoadFailed) {+ meta.logger.warn("FEPageLoadFailed encountered!!", { error: error.error });+ // TODO: what to do about this? TODOv2+ } else if (error.error instanceof AbortManagerThrownError) {+ if (error.error.tier === "engine") {+ meta.logger.warn("Engine " + error.engine + " timed out while scraping.", { error: error.error });+ } else {+ throw error.error;+ }+ } else {+ meta.logger.warn("An unexpected error happened while scraping with " + engine + ".", { error });+ }++ // Filter out the failed engine+ enginePromises = enginePromises.filter(x => x.engine !== error.engine);++ // If we don't have any engines waterfalled, let's waterfall the next engine+ if (enginePromises.length === 0) {+ break;+ }++ // Otherwise, just keep racing+ } else if (error instanceof WaterfallNextEngineSignal) {+ // It's time to waterfall the next engine+ break;+ } else if (error instanceof AbortManagerThrownError) {+ if (error.tier === "engine") {+ meta.logger.warn("Engine-scoped timeout error received here. Weird!", { error });+ }++ throw error;+ } else {+ meta.logger.warn("Unexpected error while racing engines", { error });+ throw error;+ }}}++ if (result === null) {+ meta.logger.info("Waterfalling to next engine...", {+ waitUntilWaterfall,+ });+ } else {+ break;+ }}+ snipeAbortController.abort();+if (result === null) {- if (meta.results["pdf"]?.state === "timeout") {- throw meta.results["pdf"].error ?? new TimeoutSignal();- }- if (Object.values(meta.results).every(x => x.state === "timeout")) {- throw new TimeoutSignal();- } else {+ // TODO: rectify TODOv2+ // if (meta.results["pdf"]?.state === "timeout") {+ // throw meta.results["pdf"].error ?? new TimeoutSignal();+ // }+ // if (Object.values(meta.results).every(x => x.state === "timeout")) {+ // throw new TimeoutSignal();+ // } else {throw new NoEnginesLeftError(fallbackList.map((x) => x.engine),- meta.results,+ // meta.results,);- }+ // }}let document: Document = {@@ -478,7 +523,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {numPages: result.result.numPages,contentType: result.result.contentType,proxyUsed: meta.featureFlags.has("stealthProxy") ? "stealth" : "basic",- ...((meta.results["index"] || meta.results["index;documents"]) ? (+ ...(fallbackList.find(x => ["index", "index;documents"].includes(x.engine)) ? (result.result.cacheInfo ? {cacheState: "hit",cachedAt: result.result.cacheInfo.created_at.toISOString(),@@ -506,8 +551,6 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {return {success: true,document,- logs: meta.logs,- engines: meta.results,};}@@ -536,17 +579,14 @@ export async function scrapeURL(urlToCheck,options.skipTlsVerification,meta.logger,- internalOptions.abort+ meta.abort.asSignal(),);-if (!isAllowed) {meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });return {success: false,error: new Error("URL blocked by robots.txt"),- logs: meta.logs,- engines: meta.results,};}}@@ -701,9 +741,10 @@ export async function scrapeURL(}}} catch (error) {- if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {- throw new FEPageLoadFailed();- } else if (error instanceof NoEnginesLeftError) {+ // if (Object.values(meta.results).length > 0 && Object.values(meta.results).every(x => x.state === "error" && x.error instanceof FEPageLoadFailed)) {+ // throw new FEPageLoadFailed();+ // } else+ if (error instanceof NoEnginesLeftError) {meta.logger.warn("scrapeURL: All scraping engines failed!", { error });} else if (error instanceof LLMRefusalError) {meta.logger.warn("scrapeURL: LLM refused to extract content", { error });@@ -730,6 +771,8 @@ export async function scrapeURL(meta.logger.warn("scrapeURL: Failed to prefetch PDF that is protected by anti-bot", { error });} else if (error instanceof TimeoutSignal) {throw error;+ } else if (error instanceof AbortManagerThrownError) {+ throw error.inner;} else {Sentry.captureException(error);meta.logger.error("scrapeURL: Unexpected error happened", { error });@@ -739,8 +782,6 @@ export async function scrapeURL(return {success: false,error,- logs: meta.logs,- engines: meta.results,};}}diff --git a/apps/api/src/scraper/scrapeURL/lib/abortManager.ts b/apps/api/src/scraper/scrapeURL/lib/abortManager.tsnew file mode 100644index 000000000..b9e339da7--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/abortManager.ts@@ -0,0 +1,90 @@+export type AbortInstance = {+ signal: AbortSignal;+ timesOutAt?: Date;+ tier: "external" | "scrape" | "engine";+ throwable: () => any;+}++export class AbortManager {+ private aborts: AbortInstance[] = [];+ private mappedController: AbortController | null = null;++ constructor(...instances: (AbortInstance | undefined | null)[]) {+ this.aborts = instances.filter(x => x !== undefined && x !== null) as AbortInstance[];+ }++ add(...instances: (AbortInstance | undefined | null)[]) {+ const pureInstances = instances.filter(x => x !== undefined && x !== null) as AbortInstance[];+ this.aborts.push(...pureInstances);++ if (this.mappedController !== null) {+ for (const abort of pureInstances) {+ abort.signal.addEventListener("abort", () => {+ this.mappedController?.abort(new AbortManagerThrownError(abort.tier, abort.throwable()));+ });+ }+ }+ }++ child(...instances: (AbortInstance | undefined | null)[]): AbortManager {+ const manager = new AbortManager(...this.aborts, ...instances.filter(x => x !== undefined && x !== null) as AbortInstance[]);+ return manager;+ }++ isAborted(): boolean {+ return this.aborts.some(x => x.signal.aborted);+ }++ throwIfAborted(): void {+ for (const abort of this.aborts) {+ if (abort.signal.aborted) {+ throw new AbortManagerThrownError(abort.tier, abort.throwable());+ }+ }+ }++ private _mapController() {+ this.mappedController = new AbortController();++ for (const abort of this.aborts) {+ abort.signal.addEventListener("abort", () => {+ this.mappedController?.abort(new AbortManagerThrownError(abort.tier, abort.throwable()));+ });+ }+ }++ asSignal(): AbortSignal {+ if (this.mappedController === null) {+ this._mapController();+ }++ return this.mappedController!.signal;+ }++ scrapeTimeout(): number | undefined {+ const timeouts = this.aborts.filter(x => x.tier === "scrape").map(x => x.timesOutAt).filter(x => x !== undefined);+ if (timeouts.length === 0) {+ return undefined;+ }+ return Math.min(...timeouts.map(x => x.getTime())) - Date.now();+ }++ engineNearestTimeout(): number | undefined {+ const timeouts = this.aborts.filter(x => x.tier === "engine").map(x => x.timesOutAt).filter(x => x !== undefined);+ if (timeouts.length === 0) {+ return undefined;+ }+ return Math.min(...timeouts.map(x => x.getTime())) - Date.now();+ }+}++export class AbortManagerThrownError extends Error {+ name = "AbortManagerThrownError";+ public tier: AbortInstance["tier"];+ public inner: any;+ constructor(tier: AbortInstance["tier"], inner: any) {+ super("AbortManagerThrownError: " + tier);+ this.tier = tier;+ this.inner = inner;+ }+}\ No newline at end of filediff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex a9a6188f3..99cb5aa72 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -485,14 +485,6 @@ describe("Standalone scrapeURL tests", () => {}};- // verify that log collection works properly while concurrency is happening- // expect(out.logs.length).toBeGreaterThan(0);- const weirdLogs = out.logs.filter((x) => x.scrapeId !== id);- if (weirdLogs.length > 0) {- console.warn(JSON.stringify(weirdLogs, replacer));- }- expect(weirdLogs.length).toBe(0);-if (!out.success) console.error(JSON.stringify(out, replacer));expect(out.success).toBe(true);diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 98ee0ffde..25ff35eb0 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -6,7 +6,7 @@ import {TokenUsage,} from "../../../controllers/v2/types";import { Logger } from "winston";-import { EngineResultsTracker, Meta } from "..";+import { Meta } from "..";import { logger } from "../../../lib/logger";import { modelPrices } from "../../../lib/extract/usage/model-prices";import {@@ -56,7 +56,6 @@ const getModelLimits = (model: string) => {export class LLMRefusalError extends Error {public refusal: string;- public results: EngineResultsTracker | undefined;constructor(refusal: string) {super("LLM refused to extract the website's content");diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex eee58e869..47fd4e0a5 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -316,7 +316,7 @@ export async function addScrapeJobs(export async function waitForJob(_job: Job | string,- timeout: number,+ timeout: number | null,logger: Logger = _logger,): Promise<Document> {const start = Date.now();@@ -326,18 +326,18 @@ export async function waitForJob(logger.debug("Waiting for job to be created");await new Promise(resolve => setTimeout(resolve, 500));job = await queue.getJob(_job as string);- if (Date.now() - start > timeout) {+ if (Date.now() - start > (timeout ?? 180000)) {throw new Error("Job wait (concurrency limited)");}}let doc: Document = await Promise.race([- job.waitUntilFinished(getScrapeQueueEvents(), timeout - (Date.now() - start)),- new Promise((resolve, reject) => {+ job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),+ timeout !== null ? new Promise((resolve, reject) => {setTimeout(() => {reject(new Error("Job wait " + (typeof _job == "string" ? "(was concurrency limited)" : "")));}, Math.max(0, timeout - (Date.now() - start)));- }),- ]);+ }) : null,+ ].filter(x => x !== null));logger.debug("Got job");if (!doc) {