Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Commenting one unstable Snapshotter test

Marek Trunkatade936cd71cb
diff --git a/test/autoscaling/snapshotter.js b/test/autoscaling/snapshotter.js
index 0e52af2ad..7892f732d 100644
--- a/test/autoscaling/snapshotter.js
+++ b/test/autoscaling/snapshotter.js
@@ -137,6 +137,9 @@ describe('Snapshotter', () => {
expect(overloadedCount).to.be.above(0);
});
+ /*
+ TODO: Fix this test. It's failing under load when the 2nd snapshot happens in the second half of the time
+ or the third one doesn't happen at all.
it('correctly marks memoryOverloaded', async () => {
process.env[ENV_VARS.MEMORY_MBYTES] = 20;
const options = {
@@ -145,9 +148,9 @@ describe('Snapshotter', () => {
const snapshotter = new Snapshotter(options);
await snapshotter.start();
- await Apify.utils.sleep(250);
+ await Apify.utils.sleep(199);
snapshotter.maxMemoryBytes = toBytes(1000); // Override memory to get an OK reading.
- await Apify.utils.sleep(250);
+ await Apify.utils.sleep(199);
await snapshotter.stop();
const memorySnapshots = snapshotter.getMemorySample();
@@ -157,6 +160,7 @@ describe('Snapshotter', () => {
expect(memorySnapshots[2].isOverloaded).to.be.eql(false);
delete process.env[ENV_VARS.MEMORY_MBYTES];
});
+ */
it('.get...Sample limits amount of samples', async () => {
const SAMPLE_SIZE_MILLIS = 120;