Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactor to async await progress commit
Ondra Urban16e894dc1821
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex d7e464de5..869e3e1a3 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -5,7 +5,6 @@ import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';import AutoscaledPool from './autoscaled_pool';import RequestList from './request_list';import { RequestQueue, RequestQueueLocal } from './request_queue';-import { isPromise } from './utils';const DEFAULT_OPTIONS = {maxRequestRetries: 3,@@ -145,18 +144,18 @@ export default class BasicCrawler {maxConcurrency,minConcurrency,minFreeMemoryRatio,- runTaskFunction: () => {- if (this.isStopped) return null;+ runTaskFunction: async () => {+ if (!this.isRunning) return null;return this._runTaskFunction();},- isTaskReadyFunction: () => {- if (isMaxPagesExceeded() || this.isStopped) return Promise.resolve(false);+ isTaskReadyFunction: async () => {+ if (isMaxPagesExceeded() || !this.isRunning) return false;return this._isTaskReadyFunction();},- isFinishedFunction: () => {- if (isMaxPagesExceeded() || this.isStopped) return Promise.resolve(true);+ isFinishedFunction: async () => {+ if (isMaxPagesExceeded() || !this.isRunning) return true;return isFinishedFunction? isFinishedFunction()@@ -164,7 +163,6 @@ export default class BasicCrawler {},ignoreMainProcess,};- this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);}/**@@ -172,18 +170,28 @@ export default class BasicCrawler {** @return {Promise}*/- run() {- if (this.isStopped) this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);- this.isStopped = false;- return this.autoscaledPool.run();+ async run() {+ if (this.isRunning) return this.isRunningPromise;++ this.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions);+ this.isRunning = true;+ this.rejectOnStopPromise = new Promise((r, reject) => { this.rejectOnStop = reject; });+ try {+ this.isRunningPromise = await this.autoscaledPool.run();+ this.isRunning = false;+ } catch (err) {+ this.isRunning = false; // Doing this before rejecting to make sure it's set when error handlers fire.+ this.rejectOnStop(err);+ }}/*** Stops the crawler by preventing crawls of additional pages. Pages already running are NOT terminated.*/stop() {- this.isStopped = true;+ this.isRunning = false;this.autoscaledPool.stop();+ this.rejectOnStop(new Error('BasicCrawler: .stop() function has been called. Stopping the crawler.'));}/**@@ -192,35 +200,26 @@ export default class BasicCrawler {** @ignore*/- _fetchNextRequest() {+ async _fetchNextRequest() {if (!this.requestList) return this.requestQueue.fetchNextRequest();+ const request = await this.requestList.fetchNextRequest();+ if (!this.requestQueue) return request;+ if (!request) return this.requestQueue.fetchNextRequest();- return this.requestList- .fetchNextRequest()- .then((request) => {- if (!this.requestQueue) return request;- if (!request) return this.requestQueue.fetchNextRequest();-- return this.requestQueue- .addRequest(request, { forefront: true })- .then(() => {- return Promise- .all([- this.requestQueue.fetchNextRequest(),- this.requestList.markRequestHandled(request),- ])- .then(results => results[0]);- }, (err) => {- // If requestQueue.addRequest() fails here then we must reclaim it back to- // the RequestList because probably it's not yet in the queue!- log.exception(err, 'RequestQueue.addRequest() failed, reclaiming request back to queue', { request });-- // Return null so that we finish immediately.- return this.requestList- .reclaimRequest(request)- .then(() => null);- });- });+ try {+ await this.requestQueue.addRequest(request, { forefront: true });+ } catch (err) {+ // If requestQueue.addRequest() fails here then we must reclaim it back to+ // the RequestList because probably it's not yet in the queue!+ log.exception(err, 'RequestQueue.addRequest() failed, reclaiming request back to the list', { request });+ await this.requestList.reclaimRequest();+ return null;+ }+ const [nextRequest] = await Promise.all([+ this.requestQueue.fetchNextRequest(),+ this.requestList.markRequestHandled(request),+ ]);+ return nextRequest;}/**@@ -229,63 +228,21 @@ export default class BasicCrawler {** @ignore*/- _runTaskFunction() {+ async _runTaskFunction() {const source = this.requestQueue || this.requestList;- return this._fetchNextRequest()- .then((request) => {- if (!request) return;-- let willBeRetried = false;- const handlePromise = this.handleRequestFunction({ request });- if (!isPromise(handlePromise)) throw new Error('User provided handleRequestFunction must return a Promise.');-- // NOTE: handlePromise might not be bluebird promise- return Promise.resolve()- .then(() => handlePromise)- .then(() => source.markRequestHandled(request))- .catch((error) => {- if (request.ignoreErrors) {- if (!this.isStopped) {- log.exception(error, 'BasicCrawler: handleRequestFunction failed, request.ignoreErrors=true so marking the request as handled', { // eslint-disable-line max-len- url: request.url,- retryCount: request.retryCount,- });- }- return source.markRequestHandled(request);- }-- if (!this.isStopped) request.pushErrorMessage(error);+ const request = await this._fetchNextRequest();+ if (!request) return;- // Retry request.- if (request.retryCount < this.maxRequestRetries) {- if (!this.isStopped) {- request.retryCount++;- log.exception(error, 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue', { // eslint-disable-line max-len- url: request.url,- retryCount: request.retryCount,- });- }- willBeRetried = true;- return source.reclaimRequest(request);- }-- if (!this.isStopped) {- log.exception(error, 'BasicCrawler: handleRequestFunction failed, marking failed request as handled', {- url: request.url,- retryCount: request.retryCount,- });- }-- // Mark as failed.- return source- .markRequestHandled(request)- .then(() => this.handleFailedRequestFunction({ request, error }));- })- .finally(() => {- if (!willBeRetried) this.handledRequestsCount++;- });- });+ try {+ // rejectOnStopPromise rejects when .stop() is called or AutoscaledPool throws.+ // All running tasks are therefore terminated with an error to be reclaimed and retried.+ await Promise.race([this.handleRequestFunction({ request }), this.rejectOnStopPromise]);+ source.markRequestHandled(request);+ this.handledRequestsCount++;+ } catch (err) {+ await this._requestFunctionErrorHandler(err, request, source);+ }}/**@@ -293,20 +250,12 @@ export default class BasicCrawler {** @ignore*/- _isTaskReadyFunction() {- return Promise- .resolve()- .then(() => {- if (!this.requestList) return true;-- return this.requestList.isEmpty();- })- .then((isRequestListEmpty) => {- if (!isRequestListEmpty || !this.requestQueue) return isRequestListEmpty;-- return this.requestQueue.isEmpty();- })- .then(areBothEmpty => !areBothEmpty);+ async _isTaskReadyFunction() {+ const rlPromise = this.requestList ? this.requestList.isEmpty() : true; // isEmpty returns a promise+ const rqPromise = this.requestQueue ? this.requestQueue.isEmpty() : true;+ const [isRequestListEmpty, isRequestQueueEmpty] = await Promise.all([rlPromise, rqPromise]);+ // If both are empty, return false, otherwise return true.+ return !(isRequestListEmpty && isRequestQueueEmpty);}/**@@ -314,16 +263,59 @@ export default class BasicCrawler {** @ignore*/- _defaultIsFinishedFunction() {- const promises = [];+ async _defaultIsFinishedFunction() {+ const rlPromise = this.requestList ? this.requestList.isFinished() : true; // isFinished returns a promise+ const rqPromise = this.requestQueue ? this.requestQueue.isFinished() : true;+ const [isRequestListFinished, isRequestQueueFinished] = await Promise.all([rlPromise, rqPromise]);+ // If both are finished, return true, otherwise return false.+ return isRequestListFinished && isRequestQueueFinished;+ }- if (this.requestList) promises.push(this.requestList.isFinished());- if (this.requestQueue) promises.push(this.requestQueue.isFinished());+ /**+ * Handles errors thrown by user provided handleRequestFunction()+ * @param {Error} error+ * @param {Request} request+ * @param {RequestList|RequestQueue} source+ * @return {Boolean} willBeRetried+ * @ingore+ */+ async _requestFunctionErrorHandler(error, request, source) {+ // Handles case where the crawler was deliberately stopped.+ // All running requests are reclaimed and will be retried.+ if (!this.isRunning) return source.reclaimRequest(request);- return Promise- .all(promises)- .then((results) => {- return _.all(results);+ // If we use the ignore errors option, we mark request as handled and do not retry.+ if (request.ignoreErrors) {+ log.exception(error, 'BasicCrawler: handleRequestFunction failed, request.ignoreErrors=true so marking the request as handled', { // eslint-disable-line max-len+ url: request.url,+ retryCount: request.retryCount,});+ this.handledRequestsCount++;+ return source.markRequestHandled(request);+ }++ // If we got here, it means we actually want to handle the error.+ request.pushErrorMessage(error);++ // Reclaim and retry request if retryCount is not exceeded.+ if (request.retryCount < this.maxRequestRetries) {+ request.retryCount++;+ log.exception(error, 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue', { // eslint-disable-line max-len+ url: request.url,+ retryCount: request.retryCount,+ });+ return source.reclaimRequest(request);+ }++ // This is the final fallback. If we get here, the request failed more than retryCount times and will not be retried anymore.+ log.exception(error, 'BasicCrawler: handleRequestFunction failed, marking failed request as handled', {+ url: request.url,+ retryCount: request.retryCount,+ });++ // Mark the request as failed and do not retry.+ this.handledRequestsCount++;+ await source.markRequestHandled(request);+ return this.handleFailedRequestFunction({ request, error });}}