Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix snapshotter empty array issue
Ondra Urban08ebc6e2d0b3
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex ab8e801da..138c5eea7 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -99,12 +99,6 @@ export default class Snapshotter {log.info(`Snapshotter: Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB.`);}}- // Add dummy snapshot to compare the first with.- this.eventLoopSnapshots.push({- createdAt: new Date(),- isOverloaded: false,- exceededMillis: 0,- });// Start snapshotting.this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);@@ -211,22 +205,26 @@ export default class Snapshotter {/*** Creates a snapshot of current event loop delay.* @param {Function} intervalCallback- * @return {Promise}* @ignore*/_snapshotEventLoop(intervalCallback) {try {const now = new Date();this._pruneSnapshots(this.eventLoopSnapshots, now);- const { createdAt } = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];- const delta = now - createdAt - this.eventLoopSnapshotIntervalMillis;+ // Handle empty snapshots arrayconst snapshot = {createdAt: now,isOverloaded: false,- exceededMillis: Math.max(delta - this.maxBlockedMillis, 0),+ exceededMillis: 0,};+ const previousSnapshot = this.eventLoopSnapshots[this.eventLoopSnapshots.length - 1];+ if (!previousSnapshot) return this.eventLoopSnapshots.push(snapshot);+ // Compare with previous snapshot+ const { createdAt } = previousSnapshot;+ const delta = now - createdAt - this.eventLoopSnapshotIntervalMillis;+ snapshot.exceededMillis = Math.max(delta - this.maxBlockedMillis, 0);if (delta > this.maxBlockedMillis) snapshot.isOverloaded = true;this.eventLoopSnapshots.push(snapshot);} catch (err) {diff --git a/test/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.jsindex f14a3d31e..3822b4d92 100644--- a/test/autoscaling/snapshotter.js+++ b/test/autoscaling/snapshotter.js@@ -43,7 +43,7 @@ describe('Snapshotter', () => {});expect(eventLoopSnapshots).to.be.an('array');- expect(eventLoopSnapshots).to.have.lengthOf(4);+ expect(eventLoopSnapshots).to.have.lengthOf(3);eventLoopSnapshots.forEach((ss) => {expect(ss.createdAt).to.be.a('date');expect(ss.isOverloaded).to.be.a('boolean');