Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Rewriting fetchNextRequest() to async/await
Marek Trunkat4b2e97d0ff09
diff --git a/src/request_queue.js b/src/request_queue.jsindex f0832ad46..05965964b 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -674,37 +674,36 @@ export class RequestQueueLocal {});}- fetchNextRequest() {- return this.initializationPromise- .then(() => readdirPromised(this.localPendingEmulationPath))- .then(async (files) => {- let request = null;-- while (!request && files.length) {- const filename = files.shift();- const queueOrderNo = filePathToQueueOrderNo(filename);-- if (this.queueOrderNoInProgress[queueOrderNo]) continue; // eslint-disable-line-- this.queueOrderNoInProgress[queueOrderNo] = true;- this.inProgressCount++;-- // TODO: There must be a better way. This try/catch is here because there is a race condition between- // between this and call to reclaimRequest() or markRequestHandled() that may move/rename/deleted- // the file between readdirPromised() and this function.- // Ie. the file gets listed in readdirPromised() but removed from this.queueOrderNoInProgres- // meanwhile causing this to fail.- try {- request = await this._getRequestByQueueOrderNo(queueOrderNo);- } catch (err) {- delete this.queueOrderNoInProgress[queueOrderNo];- this.inProgressCount--;- if (err.code !== 'ENOENT') throw err;- }- }+ async fetchNextRequest() {+ await this.initializationPromise;++ const files = readdirPromised(this.localPendingEmulationPath);++ let request = null;+ while (!request && files.length) {+ const filename = files.shift();+ const queueOrderNo = filePathToQueueOrderNo(filename);++ if (this.queueOrderNoInProgress[queueOrderNo]) continue; // eslint-disable-line++ this.queueOrderNoInProgress[queueOrderNo] = true;+ this.inProgressCount++;++ // TODO: There must be a better way. This try/catch is here because there is a race condition between+ // between this and call to reclaimRequest() or markRequestHandled() that may move/rename/deleted+ // the file between readdirPromised() and this function.+ // Ie. the file gets listed in readdirPromised() but removed from this.queueOrderNoInProgres+ // meanwhile causing this to fail.+ try {+ request = await this._getRequestByQueueOrderNo(queueOrderNo);+ } catch (err) {+ delete this.queueOrderNoInProgress[queueOrderNo];+ this.inProgressCount--;+ if (err.code !== 'ENOENT') throw err;+ }+ }- return request;- });+ return request;}markRequestHandled(request) {