Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added few comments

Jan Curn441fe0911697
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.js
index 6cd2918e1..d52fd5d9b 100644
--- a/src/autoscaled_pool.js
+++ b/src/autoscaled_pool.js
@@ -171,7 +171,10 @@ export default class AutoscaledPool {
this.reject = null;
// Connect to actor events for CPU info.
- // TODO: This doesn't work on local machine!
+ // TODO: This doesn't work on local machine! use blocked() both locally and on server!!!!
+ // CPU is overloaded if either blocked() or the CPU_INFO event says so
+ // the CPU status sampling should be in regular intervals (e.g. once per second),
+ // otherwise the auto-scaled pool cannot know what the data means
this.cpuInfoListener = (data) => {
this.isCpuOverloadedSnapshots = this.isCpuOverloadedSnapshots
.concat(data.isCpuOverloaded)
diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.js
index ba888855b..adca318f7 100644
--- a/test/autoscaled_pool.js
+++ b/test/autoscaled_pool.js
@@ -311,7 +311,7 @@ describe('AutoscaledPool', () => {
});
const mock = sinon.mock(utils);
- // Emit CPU overloaded events but not required emount so that we can still scale up.
+ // Emit CPU overloaded events but not required amount so that we can still scale up.
for (let i = 0; i < SCALE_DOWN_INTERVAL - 1; i++) {
Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });
}