Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
perf: improve scaling based on memory (#2459)
Martin Adámek2d5d443da5fa
diff --git a/packages/core/src/autoscaling/autoscaled_pool.ts b/packages/core/src/autoscaling/autoscaled_pool.tsindex bb4653d11..43a00da63 100644--- a/packages/core/src/autoscaling/autoscaled_pool.ts+++ b/packages/core/src/autoscaling/autoscaled_pool.ts@@ -242,8 +242,8 @@ export class AutoscaledPool {minConcurrency = 1,desiredConcurrency,desiredConcurrencyRatio = 0.90,- scaleUpStepRatio = 0.05,- scaleDownStepRatio = 0.05,+ scaleUpStepRatio = 0.1,+ scaleDownStepRatio = 0.1,maybeRunIntervalSecs = 0.5,loggingIntervalSecs = 60,taskTimeoutSecs = 0,diff --git a/packages/core/src/autoscaling/snapshotter.ts b/packages/core/src/autoscaling/snapshotter.tsindex 13a47c613..883eb2c36 100644--- a/packages/core/src/autoscaling/snapshotter.ts+++ b/packages/core/src/autoscaling/snapshotter.ts@@ -141,7 +141,7 @@ export class Snapshotter {clientSnapshotIntervalSecs = 1,snapshotHistorySecs = 30,maxBlockedMillis = 50,- maxUsedMemoryRatio = 0.7,+ maxUsedMemoryRatio = 0.9,maxClientErrors = 3,log = defaultLog,config = Configuration.getGlobalConfig(),