Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrapeURL): better timeToRun distribution

Gergő Móricz284a6ccedd1b
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 800457a86..31f7e2f2f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -203,15 +203,20 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
const results: EngineResultsTracker = {};
let result: EngineScrapeResultWithContext | null = null;
+ let ttrInstanceCount = Math.min(fallbackList.length, 3);
+ let ttrRatios = new Array(ttrInstanceCount).fill(0).map((_, i) => ttrInstanceCount - i);
+ let ttrRatioSum = ttrRatios.reduce((a, x) => a + x, 0);
+
const timeToRun = meta.options.timeout !== undefined
- ? Math.round(meta.options.timeout / Math.min(fallbackList.length, 3))
- : undefined
+ ? ttrRatios.map(ratio => Math.round(meta.options.timeout! * ratio / ttrRatioSum)).map(ratio => isNaN(ratio) ? undefined : ratio)
+ : [undefined]
- for (const { engine, unsupportedFeatures } of fallbackList) {
+ for (const i in fallbackList) {
+ const { engine, unsupportedFeatures } = fallbackList[i];
const startedAt = Date.now();
try {
meta.logger.info("Scraping via " + engine + "...");
- const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun);
+ const _engineResult = await scrapeURLWithEngine(meta, engine, timeToRun[i] ?? timeToRun.slice(-1)[0]);
if (_engineResult.markdown === undefined) {
// Some engines emit Markdown directly.
_engineResult.markdown = await parseMarkdown(_engineResult.html);