Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(core): respect `systemInfoV2` in snapshotter (#2961)

Martin Adámek4100eabf171d
diff --git a/packages/core/src/autoscaling/snapshotter.ts b/packages/core/src/autoscaling/snapshotter.ts
index 866c2199e..9792bc2c3 100644
--- a/packages/core/src/autoscaling/snapshotter.ts
+++ b/packages/core/src/autoscaling/snapshotter.ts
@@ -1,5 +1,5 @@
import type { StorageClient } from '@crawlee/types';
-import { getMemoryInfo } from '@crawlee/utils';
+import { getMemoryInfo, getMemoryInfoV2, isContainerized } from '@crawlee/utils';
import ow from 'ow';
import type { Log } from '@apify/log';
@@ -195,7 +195,17 @@ export class Snapshotter {
if (memoryMbytes > 0) {
this.maxMemoryBytes = memoryMbytes * 1024 * 1024;
} else {
- const { totalBytes } = await getMemoryInfo();
+ let totalBytes: number;
+
+ if (this.config.get('systemInfoV2')) {
+ const containerized = this.config.get('containerized', await isContainerized());
+ const memInfo = await getMemoryInfoV2(containerized);
+ totalBytes = memInfo.totalBytes;
+ } else {
+ const memInfo = await getMemoryInfo();
+ totalBytes = memInfo.totalBytes;
+ }
+
this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio')!);
this.log.debug(
`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
diff --git a/packages/core/src/events/local_event_manager.ts b/packages/core/src/events/local_event_manager.ts
index 9f54c403d..27ca2eeb7 100644
--- a/packages/core/src/events/local_event_manager.ts
+++ b/packages/core/src/events/local_event_manager.ts
@@ -54,11 +54,7 @@ export class LocalEventManager extends EventManager {
* @internal
*/
async isContainerizedWrapper() {
- const config = this.config.get('containerized');
- if (config !== undefined) {
- return config;
- }
- return isContainerized();
+ return this.config.get('containerized', await isContainerized());
}
private getCurrentCpuTicks() {