Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Tweaking autoscaled pool values
Marek Trunkat3c10dce8f3d0
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex 13851568a..13267f77a 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -8,7 +8,7 @@ import SystemStatus from './system_status';const DEFAULT_OPTIONS = {maxConcurrency: 1000,minConcurrency: 1,- desiredConcurrencyRatio: 0.95,+ desiredConcurrencyRatio: 0.90,scaleUpStepRatio: 0.05,scaleDownStepRatio: 0.05,maybeRunIntervalSecs: 0.5,diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex 8aab6fc18..ab8e801da 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -8,10 +8,10 @@ import events from '../events';const DEFAULT_OPTIONS = {eventLoopSnapshotIntervalSecs: 0.5,- maxBlockedMillis: 50,+ maxBlockedMillis: 50, // 0.05memorySnapshotIntervalSecs: 1,maxUsedMemoryRatio: 0.7,- snapshotHistorySecs: 60,+ snapshotHistorySecs: 30,};/**diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.jsindex 1af5c09d5..0eae8f2eb 100644--- a/src/autoscaling/system_status.js+++ b/src/autoscaling/system_status.js@@ -7,7 +7,7 @@ const DEFAULT_OPTIONS = {currentHistorySecs: 5, // TODO this should be something like "nowDurationSecs" but it's weird, ideas?maxMemoryOverloadedRatio: 0.2,maxEventLoopOverloadedRatio: 0.2,- maxCpuOverloadedRatio: 0.1,+ maxCpuOverloadedRatio: 0.2,};/**