Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Progress commit

Ondra Urban47858f97b41b
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 869e3e1a3..fbe2c21ea 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -176,8 +176,9 @@ export default class BasicCrawler {
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
this.isRunning = true;
this.rejectOnStopPromise = new Promise((r, reject) => { this.rejectOnStop = reject; });
+ this.isRunningPromise = this.autoscaledPool.run();
try {
- this.isRunningPromise = await this.autoscaledPool.run();
+ await this.isRunningPromise;
this.isRunning = false;
} catch (err) {
this.isRunning = false; // Doing this before rejecting to make sure it's set when error handlers fire.
@@ -251,9 +252,13 @@ export default class BasicCrawler {
* @ignore
*/
async _isTaskReadyFunction() {
- const rlPromise = this.requestList ? this.requestList.isEmpty() : true; // isEmpty returns a promise
- const rqPromise = this.requestQueue ? this.requestQueue.isEmpty() : true;
- const [isRequestListEmpty, isRequestQueueEmpty] = await Promise.all([rlPromise, rqPromise]);
+ const [
+ isRequestListEmpty,
+ isRequestQueueEmpty,
+ ] = await Promise.all([
+ this.requestList ? this.requestList.isEmpty() : true,
+ this.requestQueue ? this.requestQueue.isEmpty() : true,
+ ]);
// If both are empty, return false, otherwise return true.
return !(isRequestListEmpty && isRequestQueueEmpty);
}
@@ -264,9 +269,13 @@ export default class BasicCrawler {
* @ignore
*/
async _defaultIsFinishedFunction() {
- const rlPromise = this.requestList ? this.requestList.isFinished() : true; // isFinished returns a promise
- const rqPromise = this.requestQueue ? this.requestQueue.isFinished() : true;
- const [isRequestListFinished, isRequestQueueFinished] = await Promise.all([rlPromise, rqPromise]);
+ const [
+ isRequestListFinished,
+ isRequestQueueFinished,
+ ] = await Promise.all([
+ this.requestList ? this.requestList.isFinished() : true,
+ this.requestQueue ? this.requestQueue.isFinished() : true,
+ ]);
// If both are finished, return true, otherwise return false.
return isRequestListFinished && isRequestQueueFinished;
}
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 2f4d38ec2..473b7f574 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -39,14 +39,16 @@ describe('BasicCrawler', () => {
});
it('should run, stop and then finish', async () => {
- const sources = _.range(500).map(index => ({ url: `https://example.com/${index}` }));
+ const sources = _.range(1, 500).map(index => ({ url: `https://example.com/${index}` }));
let basicCrawler;
+ let isStopped;
const processed = [];
const requestList = new Apify.RequestList({ sources });
const handleRequestFunction = async ({ request }) => {
if (request.url.endsWith('200')) {
- basicCrawler.stop();
+ if (!isStopped) basicCrawler.stop();
+ isStopped = true;
}
await delayPromise(10);
processed.push(_.pick(request, 'url'));
@@ -64,13 +66,13 @@ describe('BasicCrawler', () => {
// The crawler will stop after 200 requests
await basicCrawler.run();
- expect(processed.length).to.be.above(175);
- expect(processed.length).to.be.below(201);
+ expect(processed.length).to.be.eql(175);
expect(await requestList.isFinished()).to.be.eql(false);
expect(await requestList.isEmpty()).to.be.eql(false);
await basicCrawler.run();
- expect(processed).to.be.eql(sources);
+ expect(processed.length).to.be.within(500, 525);
+ expect(new Set(processed.map(p => p.url))).to.be.eql(new Set(sources.map(s => s.url)));
expect(await requestList.isFinished()).to.be.eql(true);
expect(await requestList.isEmpty()).to.be.eql(true);
});
@@ -357,11 +359,14 @@ describe('BasicCrawler', () => {
expect(processed['http://example.com/2'].retryCount).to.be.eql(0);
expect(processed['http://example.com/1'].userData.foo).to.be.a('undefined');
- expect(processed['http://example.com/1'].errorMessages).to.have.lengthOf(4);
- expect(processed['http://example.com/1'].retryCount).to.be.eql(3);
+ expect(processed['http://example.com/1'].errorMessages).to.have.lengthOf(2);
+ expect(processed['http://example.com/1'].retryCount).to.be.eql(2);
expect(await requestList.isFinished()).to.be.eql(true);
expect(await requestList.isEmpty()).to.be.eql(true);
+
+ mock.verify();
+ mock.restore();
});
it('should say that task is not ready requestList is not set and requestQueue is empty', async () => {
@@ -412,6 +417,9 @@ describe('BasicCrawler', () => {
await basicCrawler.run();
expect(processed).to.be.eql([request0, request1]);
+
+ mock.verify();
+ mock.restore();
});
it('should support maxRequestsPerCrawl parameter', async () => {