Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "fix(scrapeURL): better timeToRun distribution"
Gergő Móricz7f57c868be83
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 31f7e2f2f..800457a86 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -203,20 +203,15 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {const results: EngineResultsTracker = {};let result: EngineScrapeResultWithContext | null = null;- let ttrInstanceCount = Math.min(fallbackList.length, 3);- let ttrRatios = new Array(ttrInstanceCount).fill(0).map((_, i) => ttrInstanceCount - i);- let ttrRatioSum = ttrRatios.reduce((a, x) => a + x, 0);-const timeToRun = meta.options.timeout !== undefined- ? ttrRatios.map(ratio => Math.round(meta.options.timeout! * ratio / ttrRatioSum)).map(ratio => isNaN(ratio) ? undefined : ratio)- : [undefined]+ ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 3))+ : undefined- for (const i in fallbackList) {- const { engine, unsupportedFeatures } = fallbackList[i];+ for (const { engine, unsupportedFeatures } of fallbackList) {const startedAt = Date.now();try {meta.logger.info("Scraping via " + engine + "...");- const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun[i] ?? timeToRun.slice(-1)[0]);+ const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun);if (_engineResult.markdown === undefined) {// Some engines emit Markdown directly._engineResult.markdown = await parseMarkdown(_engineResult.html);