Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL): better timeToRun distribution
Gergő Móricz284a6ccedd1b
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 800457a86..31f7e2f2f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -203,15 +203,20 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const results: EngineResultsTracker = {};let result: EngineScrapeResultWithContext | null = null;+ let ttrInstanceCount = Math.min(fallbackList.length, 3);+ let ttrRatios = new Array(ttrInstanceCount).fill(0).map((_, i) => ttrInstanceCount - i);+ let ttrRatioSum = ttrRatios.reduce((a, x) => a + x, 0);+const timeToRun = meta.options.timeout !== undefined- ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 3))- : undefined+ ? ttrRatios.map(ratio => Math.round(meta.options.timeout! * ratio / ttrRatioSum)).map(ratio => isNaN(ratio) ? undefined : ratio)+ : [undefined]- for (const { engine, unsupportedFeatures } of fallbackList) {+ for (const i in fallbackList) {+ const { engine, unsupportedFeatures } = fallbackList[i];const startedAt = Date.now();try {meta.logger.info("Scraping via " + engine + "...");- const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun);+ const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun[i] ?? timeToRun.slice(-1)[0]);if (_engineResult.markdown === undefined) {// Some engines emit Markdown directly._engineResult.markdown = await parseMarkdown(_engineResult.html);