Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fixing race condition in request queue
Marek Trunkat5320a940b2d5
diff --git a/src/request_queue.js b/src/request_queue.jsindex 1a2e59116..3b0b92e32 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -678,24 +678,33 @@ export class RequestQueueLocal {fetchNextRequest() {return this.initializationPromise.then(() => readdirPromised(this.localPendingEmulationPath))- .then((files) => {- let queueOrderNo;-- _.find(files, (filename) => {- const no = filePathToQueueOrderNo(filename);-- if (!this.queueOrderNoInProgress[no]) {- queueOrderNo = no;- return true;+ .then(async (files) => {+ let request = null;++ while (!request && files.length) {+ const filename = files.shift();+ const queueOrderNo = filePathToQueueOrderNo(filename);++ if (this.queueOrderNoInProgress[queueOrderNo]) continue; // eslint-disable-line++ this.queueOrderNoInProgress[queueOrderNo] = true;+ this.inProgressCount++;++ // TODO: There must be a better way. This try/catch is here because there is a race condition between+ // between this and call to reclaimRequest() or markRequestHandled() that may move/rename/deleted+ // the file between readdirPromised() and this function.+ // Ie. the file gets listed in readdirPromised() but removed from this.queueOrderNoInProgres+ // meanwhile causing this to fail.+ try {+ request = await this._getRequestByQueueOrderNo(queueOrderNo);+ } catch (err) {+ delete this.queueOrderNoInProgress[queueOrderNo];+ this.inProgressCount--;+ if (err.code !== 'ENOENT') throw err;}- });-- if (!queueOrderNo) return null;-- this.queueOrderNoInProgress[queueOrderNo] = true;- this.inProgressCount++;+ }- return this._getRequestByQueueOrderNo(queueOrderNo);+ return request;});}@@ -714,10 +723,11 @@ export class RequestQueueLocal {if (!request.handledAt) request.handledAt = new Date();- return Promise.all([- writeFilePromised(dest, JSON.stringify(request, null, 4)),- deleteFilePromised(source),- ])+ return Promise+ .all([+ writeFilePromised(dest, JSON.stringify(request, null, 4)),+ deleteFilePromised(source),+ ]).then(() => {this.pendingCount--;this.inProgressCount--;@@ -745,10 +755,11 @@ export class RequestQueueLocal {this.requestIdToQueueOrderNo[request.id] = newQueueOrderNo;- return Promise.all([- writeFilePromised(this._getFilePath(newQueueOrderNo), JSON.stringify(request, null, 4)),- deleteFilePromised(this._getFilePath(oldQueueOrderNo)),- ])+ return Promise+ .all([+ writeFilePromised(this._getFilePath(newQueueOrderNo), JSON.stringify(request, null, 4)),+ deleteFilePromised(this._getFilePath(oldQueueOrderNo)),+ ]).then(() => {this.inProgressCount--;delete this.queueOrderNoInProgress[oldQueueOrderNo];