Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): reintroduce default timeout for simple queries (#1440)
Gergő Móricz4294face784e
diff --git a/apps/api/src/__tests__/snips/batch-scrape.test.ts b/apps/api/src/__tests__/snips/batch-scrape.test.tsindex 1890b08bf..f3e9e585b 100644--- a/apps/api/src/__tests__/snips/batch-scrape.test.ts+++ b/apps/api/src/__tests__/snips/batch-scrape.test.ts@@ -8,7 +8,7 @@ describe("Batch scrape tests", () => {expect(response.body.data[0]).toHaveProperty("markdown");expect(response.body.data[0].markdown).toContain("Firecrawl");- }, 30000);+ }, 180000);if (!process.env.TEST_SUITE_SELF_HOSTED) {describe("JSON format", () => {@@ -45,7 +45,7 @@ describe("Batch scrape tests", () => {expect(response.body.data[0].json).toHaveProperty("is_open_source");expect(response.body.data[0].json.is_open_source).toBe(true);expect(typeof response.body.data[0].json.is_open_source).toBe("boolean");- }, 30000);+ }, 180000);});}});diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex e047d1cb6..cedd275e2 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -230,7 +230,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const timeToRun =meta.options.timeout !== undefined? Math.round(meta.options.timeout / Math.min(fallbackList.length, 2))- : undefined;+ : (!meta.options.actions && !meta.options.jsonOptions && !meta.options.extract)+ ? Math.round(120000 / Math.min(fallbackList.length, 2))+ : undefined;for (const { engine, unsupportedFeatures } of fallbackList) {meta.internalOptions.abort?.throwIfAborted();