Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add autoscaled pool test

Ondra Urban16fe7870ece3
diff --git a/test/autoscaling/autoscaled_pool.js b/test/autoscaling/autoscaled_pool.js
index 7e1fac5c6..ba3caf4aa 100644
--- a/test/autoscaling/autoscaled_pool.js
+++ b/test/autoscaling/autoscaled_pool.js
@@ -342,14 +342,13 @@ describe('AutoscaledPool', () => {
let finished = false;
let aborted = false;
const pool = new AutoscaledPool({
- runTaskFunction: () => {
+ runTaskFunction: async () => {
if (!aborted) {
- return (async () => {
- await pool.abort();
- aborted = true;
- })();
+ await pool.abort();
+ aborted = true;
+ } else {
+ return null;
}
- return null;
},
isFinishedFunction: () => { finished = true; },
isTaskReadyFunction: () => !aborted,
@@ -382,6 +381,42 @@ describe('AutoscaledPool', () => {
expect(started).to.be.eql(true);
expect(completed).to.be.eql(true);
});
+
+ it('should pause and resume', async () => {
+ let count = 0;
+ const results = [];
+ let pauseResolve;
+ const pausePromise = new Promise((res) => { pauseResolve = res; });
+
+ const pool = new AutoscaledPool({
+ maybeRunIntervalSecs: 0.01,
+ minConcurrency: 10,
+ runTaskFunction: async () => {
+ results.push(count++);
+ if (count === 20) {
+ pool.pause().then(pauseResolve);
+ }
+ },
+ isFinishedFunction: async () => !(count < 50),
+ isTaskReadyFunction: async () => count < 50,
+ });
+
+ let finished = false;
+ const runPromise = pool.run();
+ runPromise.then(() => { finished = true; });
+ await pausePromise;
+ expect(count).to.be.eql(20);
+ expect(finished).to.be.eql(false);
+ expect(results).to.have.lengthOf(count);
+ results.forEach((r, i) => expect(r).to.be.eql(i));
+
+ pool.resume();
+ await runPromise;
+ expect(count).to.be.eql(50);
+ expect(finished).to.be.eql(true);
+ expect(results).to.have.lengthOf(count);
+ results.forEach((r, i) => expect(r).to.be.eql(i));
+ });
});
/* eslint-enable no-underscore-dangle */