Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix unstable tests
Ondra Urban471fc78c4cb4
diff --git a/test/autoscaling/autoscaled_pool.js b/test/autoscaling/autoscaled_pool.jsindex 19b42f0d9..7cedbeb32 100644--- a/test/autoscaling/autoscaled_pool.js+++ b/test/autoscaling/autoscaled_pool.js@@ -253,62 +253,29 @@ describe('AutoscaledPool', () => {});it('should break and resume when the task queue is empty for a while', async () => {- const tasks = [];const finished = [];-let isFinished = false;let isTaskReady = true;- const getTask = (id) => {- return () => delayPromise(10).then(() => finished.push(id));- };-- // Start 3 tasks immediately.- tasks.push(getTask(0));- tasks.push(getTask(1));- setTimeout(() => tasks.push(getTask(2)), 20);-- setTimeout(() => {- isTaskReady = false;- }, 50);-- // Add 2 tasks after 80ms.- setTimeout(() => tasks.push(getTask(3)), 80);- setTimeout(() => tasks.push(getTask(4)), 80);-- // Remove tasks and set isTaskReady=true to accept a new task.- setTimeout(() => {- tasks.pop();- tasks.pop();- isTaskReady = true;- }, 120);-- // Add one more task.- setTimeout(() => tasks.push(getTask(5)), 150);-- // Finish.- setTimeout(() => {- isFinished = true;- }, 200);-- // Run the pool and close it after 3s.+ let counter = 0;const pool = new AutoscaledPool({- minConcurrency: 3,- runTaskFunction: () => {- const task = tasks.shift();-- if (!task) return;-- return task();+ maxConcurrency: 1,+ runTaskFunction: async () => {+ await delayPromise(1);+ if (counter === 10) { isTaskReady = false; setTimeout(() => { isTaskReady = true; }, 10); }+ if (counter === 19) { isTaskReady = false; isFinished = true; }+ counter++;+ finished.push(Date.now());},- isFinishedFunction: () => Promise.resolve(isFinished),- isTaskReadyFunction: () => Promise.resolve(!isFinished && isTaskReady),+ isFinishedFunction: async () => isFinished,+ isTaskReadyFunction: async () => !isFinished && isTaskReady,});- pool.maybeRunIntervalMillis = 5;+ pool.maybeRunIntervalMillis = 1;await pool.run();// Check finished tasks.- expect(finished).to.be.eql([0, 1, 2, 5]);+ expect(finished).to.have.lengthOf(20);+ expect(finished[11] - finished[10]).to.be.above(9);});it('should work with loggingIntervalMillis = null', async () => {diff --git a/test/autoscaling/system_status.js b/test/autoscaling/system_status.jsindex d940fdeb9..4f38ff383 100644--- a/test/autoscaling/system_status.js+++ b/test/autoscaling/system_status.js@@ -36,9 +36,10 @@ describe('SystemStatus', () => {const generateSnapsSync = (percentage, overloaded) => {const snaps = [];+ const createdAt = new Date();for (let i = 0; i < 100; i++) {snaps.push({- createdAt: new Date(),+ createdAt,isOverloaded: i < percentage ? overloaded : !overloaded,});}