Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Optimize isTaskReadyFunction
Ondra Urbanb5d55bdb2f96
diff --git a/src/basic_crawler.js b/src/basic_crawler.jsindex 5657e5a7e..e6a50113b 100644--- a/src/basic_crawler.js+++ b/src/basic_crawler.js@@ -249,20 +249,17 @@ export default class BasicCrawler {}/**- * Returns true if some RequestList and RequestQueue have request ready for processing.+ * Returns true if either RequestList or RequestQueue have a request ready for processing.** @ignore*/async _isTaskReadyFunction() {- const [- isRequestListEmpty,- isRequestQueueEmpty,- ] = await Promise.all([- this.requestList ? this.requestList.isEmpty() : true,- this.requestQueue ? this.requestQueue.isEmpty() : true,- ]);- // If both are empty, return false, otherwise return true.- return !(isRequestListEmpty && isRequestQueueEmpty);+ // First check RequestList, since it's only in memory.+ const isRequestListEmpty = this.requestList ? (await this.requestList.isEmpty()) : true;+ // If RequestList is not empty, task is ready, no reason to check RequestQueue.+ if (!isRequestListEmpty) return true;+ // If RequestQueue is not empty, task is ready, return true, otherwise false.+ return this.requestQueue ? !(await this.requestQueue.isEmpty()) : false;}/**diff --git a/test/basic_crawler.js b/test/basic_crawler.jsindex 0b7105c27..8c0e2171a 100644--- a/test/basic_crawler.js+++ b/test/basic_crawler.js@@ -343,7 +343,7 @@ describe('BasicCrawler', () => {.returns(Promise.resolve());mock.expects('isEmpty')- .exactly(6)+ .exactly(3).returns(Promise.resolve(false));mock.expects('isEmpty').once()