Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Implement, add test

Ondra Urban89f5035fb0af
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index 29dbc29ba..8c4c2d67e 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -5,6 +5,7 @@ import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import AutoscaledPool from './autoscaling/autoscaled_pool';
import RequestList from './request_list';
import { RequestQueue, RequestQueueLocal } from './request_queue';
+import { apifyClient } from './utils';
const DEFAULT_OPTIONS = {
maxRequestRetries: 3,
@@ -192,6 +193,7 @@ class BasicCrawler {
async run() {
if (this.isRunning) return this.isRunningPromise;
+ await this._loadHandledRequestCount();
this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);
this.isRunning = true;
this.rejectOnAbortPromise = new Promise((r, reject) => { this.rejectOnAbort = reject; });
@@ -345,6 +347,26 @@ class BasicCrawler {
await source.markRequestHandled(request);
return this.handleFailedRequestFunction({ request, error }); // This function prints an error message.
}
+
+ /**
+ * Updates handledRequestsCount from possibly stored counts,
+ * usually after worker migration. Since one of the stores
+ * needs to have priority when both are present,
+ * it is the request queue, because generally, the request
+ * list will first be dumped into the queue and then left
+ * empty.
+ *
+ * @return {Promise}
+ * @ignore
+ */
+ async _loadHandledRequestCount() {
+ if (this.requestQueue) {
+ const queueInfo = await apifyClient.requestQueues.getQueue({ queueId: this.requestQueue.queueId });
+ this.handledRequestsCount = queueInfo.handledRequestCount;
+ } else if (this.requestList) {
+ this.handledRequestsCount = this.requestList.handledCount();
+ }
+ }
}
export default BasicCrawler;
diff --git a/test/basic_crawler.js b/test/basic_crawler.js
index 75aed4f0e..533402f68 100644
--- a/test/basic_crawler.js
+++ b/test/basic_crawler.js
@@ -488,4 +488,91 @@ describe('BasicCrawler', () => {
expect(await requestList.isFinished()).to.be.eql(false);
expect(await requestList.isEmpty()).to.be.eql(false);
});
+
+ it('should load handledRequestCount from storages', async () => {
+ const requestQueue = new RequestQueue('id');
+ requestQueue.isEmpty = async () => false;
+ requestQueue.isFinished = async () => false;
+ requestQueue.fetchNextRequest = async () => (new Apify.Request({ id: 'id', url: 'http://example.com' }));
+ requestQueue.markRequestHandled = async () => {};
+ let stub = sinon
+ .stub(Apify.client.requestQueues, 'getQueue')
+ .returns(Promise.resolve({
+ handledRequestCount: 33,
+ }));
+
+ let count = 0;
+ let crawler = new Apify.BasicCrawler({
+ requestQueue,
+ maxConcurrency: 1,
+ handleRequestFunction: async () => {
+ await delayPromise(1);
+ count++;
+ },
+ maxRequestsPerCrawl: 40,
+ });
+
+ await crawler.run();
+ sinon.assert.calledWith(stub, sinon.match({ queueId: 'id' }));
+ expect(count).to.be.eql(7);
+ sinon.restore();
+
+ const sources = _.range(1, 10).map(i => ({ url: `http://example.com/${i}` }));
+ let requestList = new Apify.RequestList({ sources });
+ await requestList.initialize();
+ stub = sinon
+ .stub(requestList, 'handledCount')
+ .returns(33);
+
+ count = 0;
+ crawler = new Apify.BasicCrawler({
+ requestList,
+ maxConcurrency: 1,
+ handleRequestFunction: async () => {
+ await delayPromise(1);
+ count++;
+ },
+ maxRequestsPerCrawl: 40,
+ });
+
+ await crawler.run();
+ sinon.assert.called(stub);
+ expect(count).to.be.eql(7);
+ sinon.restore();
+
+ requestList = new Apify.RequestList({ sources });
+ await requestList.initialize();
+ const listStub = sinon
+ .stub(requestList, 'handledCount')
+ .returns(20);
+
+ const queueStub = sinon
+ .stub(Apify.client.requestQueues, 'getQueue')
+ .returns(Promise.resolve({
+ handledRequestCount: 33,
+ }));
+
+ const addRequestStub = sinon
+ .stub(requestQueue, 'addRequest')
+ .returns(Promise.resolve());
+
+ count = 0;
+ crawler = new Apify.BasicCrawler({
+ requestList,
+ requestQueue,
+ maxConcurrency: 1,
+ handleRequestFunction: async () => {
+ await delayPromise(1);
+ count++;
+ },
+ maxRequestsPerCrawl: 40,
+ });
+
+ await crawler.run();
+ sinon.assert.calledWith(queueStub, sinon.match({ queueId: 'id' }));
+ sinon.assert.notCalled(listStub);
+ sinon.assert.callCount(addRequestStub, 7);
+ expect(count).to.be.eql(7);
+ sinon.restore();
+ });
});