Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix RequestQueueLocal race condition

Ondra Urban7c7a79fb8eb2
diff --git a/src/request_queue.js b/src/request_queue.js
index 397a0fa77..ba0053634 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -1,6 +1,5 @@
import path from 'path';
import crypto from 'crypto';
-import { promisify } from 'util';
import fs from 'fs-extra';
import { checkParamOrThrow } from 'apify-client/build/utils';
import LruCache from 'apify-shared/lru_cache';
@@ -34,14 +33,6 @@ const RECENTLY_HANDLED_CACHE_SIZE = 1000;
// to be available to subsequent reads.
export const STORAGE_CONSISTENCY_DELAY_MILLIS = 3000;
-
-const writeFilePromised = promisify(fs.writeFile);
-const readdirPromised = promisify(fs.readdir);
-const readFilePromised = promisify(fs.readFile);
-const renamePromised = promisify(fs.rename);
-const statPromised = promisify(fs.stat);
-const emptyDirPromised = promisify(fs.emptyDir);
-
const { requestQueues } = apifyClient;
const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queues are stored here.
@@ -712,6 +703,8 @@ export class RequestQueue {
}
}
+const ENOENT = 'ENOENT';
+
/**
* Helper function that extracts queue order number from filename.
*
@@ -763,8 +756,8 @@ export class RequestQueueLocal {
await ensureDirExists(this.localPendingEmulationPath);
const [handled, pending] = await Promise.all([
- readdirPromised(this.localHandledEmulationPath),
- readdirPromised(this.localPendingEmulationPath),
+ fs.readdir(this.localHandledEmulationPath),
+ fs.readdir(this.localPendingEmulationPath),
]);
this.pendingCount = pending.length;
@@ -774,7 +767,7 @@ export class RequestQueueLocal {
const pendingPaths = pending.map(filename => path.join(this.localPendingEmulationPath, filename));
const filePaths = handledPaths.concat(pendingPaths);
- const stats = await statPromised(this.localPendingEmulationPath);
+ const stats = await fs.stat(this.localPendingEmulationPath);
this.createdAt = stats.birthtime;
this.modifiedAt = stats.mtime;
this.accessedAt = stats.atime;
@@ -785,7 +778,7 @@ export class RequestQueueLocal {
}
async _saveRequestIdToQueueOrderNo(filepath) {
- const str = await readFilePromised(filepath);
+ const str = await fs.readFile(filepath);
const request = JSON.parse(str);
this.requestIdToQueueOrderNo[request.id] = filePathToQueueOrderNo(filepath);
}
@@ -804,29 +797,33 @@ export class RequestQueueLocal {
const base = (10 ** (13)); // Date.now() returns int with 13 numbers.
// We always add pending count for a case that two pages are insterted at the same millisecond.
const now = Date.now() + this.queueOrderNoCounter++;
- const queueOrderNo = forefront
+ return forefront
? sgn + (base - now)
: sgn + (base + now);
-
- return queueOrderNo;
}
- _getRequestByQueueOrderNo(queueOrderNo) {
+ async _getRequestByQueueOrderNo(queueOrderNo) {
checkParamOrThrow(queueOrderNo, 'queueOrderNo', 'Number');
-
- return readFilePromised(this._getFilePath(queueOrderNo, false))
- .catch((err) => {
- if (err.code !== 'ENOENT') throw err;
-
- return readFilePromised(this._getFilePath(queueOrderNo, true));
- })
- .then((str) => {
- if (!str) throw new Error('Request was not found in none of handled and pending directories!');
-
- const obj = JSON.parse(str);
-
- return new Request(obj);
- });
+ let buffer;
+ try {
+ const pendingPath = this._getFilePath(queueOrderNo);
+ buffer = await fs.readFile(pendingPath);
+ } catch (err) {
+ if (err.code !== ENOENT) throw err;
+ const handledPath = this._getFilePath(queueOrderNo, true);
+ buffer = await fs.readFile(handledPath);
+ }
+ const json = buffer.toString();
+ if (!json) {
+ // This happens when the file is still being written.
+ // The descriptor exists, but the contents are not there yet.
+ // So let's handle it as if it didn't exist at all.
+ const error = new Error(ENOENT);
+ error.code = ENOENT;
+ throw error;
+ }
+ const requestOptions = JSON.parse(json);
+ return new Request(requestOptions);
}
async addRequest(request, opts = {}) {
@@ -834,65 +831,65 @@ export class RequestQueueLocal {
request.uniqueKey = newRequest.uniqueKey;
- return this.initializationPromise
- .then(() => {
- const queueOrderNo = this._getQueueOrderNo(forefront);
-
- // Add ID as server does.
- // TODO: This way of cloning doesn't preserve Dates!
- const requestCopy = JSON.parse(JSON.stringify(newRequest));
- requestCopy.id = getRequestId(request.uniqueKey);
- request.id = requestCopy.id;
-
- this._updateMetadata(true);
-
- // If request already exists then don't override it!
- if (this.requestIdToQueueOrderNo[requestCopy.id]) {
- return this
- .getRequest(requestCopy.id)
- .then(existingRequest => ({
- requestId: existingRequest.id,
- wasAlreadyHandled: existingRequest && existingRequest.handledAt,
- wasAlreadyPresent: true,
- request,
- }));
- }
-
- this.requestIdToQueueOrderNo[requestCopy.id] = queueOrderNo;
- if (!requestCopy.handledAt) this.pendingCount++;
-
- const filePath = this._getFilePath(queueOrderNo, !!requestCopy.handledAt);
-
- return writeFilePromised(filePath, JSON.stringify(requestCopy, null, 4))
- .then(() => ({
- requestId: requestCopy.id,
- wasAlreadyHandled: false,
- wasAlreadyPresent: false,
- request,
- }));
- });
+ await this.initializationPromise;
+ const queueOrderNo = this._getQueueOrderNo(forefront);
+
+ // Add ID as server does.
+ // TODO: This way of cloning doesn't preserve Dates!
+ const requestCopy = JSON.parse(JSON.stringify(newRequest));
+ requestCopy.id = getRequestId(request.uniqueKey);
+ request.id = requestCopy.id;
+
+ this._updateMetadata(true);
+
+ // If request already exists then don't override it!
+ if (this.requestIdToQueueOrderNo[requestCopy.id]) {
+ const existingRequest = await this.getRequest(requestCopy.id);
+ return {
+ requestId: existingRequest.id,
+ wasAlreadyHandled: existingRequest && existingRequest.handledAt,
+ wasAlreadyPresent: true,
+ request,
+ };
+ }
+
+ if (!requestCopy.handledAt) this.pendingCount++;
+
+ const filePath = this._getFilePath(queueOrderNo, !!requestCopy.handledAt);
+
+ // Lock the file until we fully dump data!
+ // This is to avoid a race condition where readdir would return a path
+ // to the file, but the file would still be empty and subsequent read would fail.
+
+ await fs.writeFile(filePath, JSON.stringify(requestCopy, null, 4));
+
+ this.requestIdToQueueOrderNo[requestCopy.id] = queueOrderNo;
+ return {
+ requestId: requestCopy.id,
+ wasAlreadyHandled: false,
+ wasAlreadyPresent: false,
+ request,
+ };
}
- getRequest(requestId) {
+ async getRequest(requestId) {
validateGetRequestParams(requestId);
- return this.initializationPromise
- .then(() => {
- const queueOrderNo = this.requestIdToQueueOrderNo[requestId];
+ await this.initializationPromise;
+ const queueOrderNo = this.requestIdToQueueOrderNo[requestId];
- this._updateMetadata();
+ this._updateMetadata();
- // TODO: We should update the last access time to files...
- if (!queueOrderNo) return null;
+ // TODO: We should update the last access time to files...
+ if (!queueOrderNo) return null;
- return this._getRequestByQueueOrderNo(queueOrderNo);
- });
+ return this._getRequestByQueueOrderNo(queueOrderNo);
}
async fetchNextRequest() {
await this.initializationPromise;
- const files = await readdirPromised(this.localPendingEmulationPath);
+ const files = await fs.readdir(this.localPendingEmulationPath);
this._updateMetadata();
@@ -908,92 +905,84 @@ export class RequestQueueLocal {
// TODO: There must be a better way. This try/catch is here because there is a race condition between
// between this and call to reclaimRequest() or markRequestHandled() that may move/rename/deleted
- // the file between readdirPromised() and this function.
- // Ie. the file gets listed in readdirPromised() but removed from this.queueOrderNoInProgres
+ // the file between fs.readdir() and this function.
+ // Ie. the file gets listed in fs.readdir() but removed from this.queueOrderNoInProgress
// meanwhile causing this to fail.
try {
request = await this._getRequestByQueueOrderNo(queueOrderNo);
} catch (err) {
delete this.queueOrderNoInProgress[queueOrderNo];
this.inProgressCount--;
- if (err.code !== 'ENOENT') throw err;
+ if (err.code !== ENOENT) throw err;
}
}
return request;
}
- markRequestHandled(request) {
+ async markRequestHandled(request) {
validateMarkRequestHandledParams(request);
- return this.initializationPromise
- .then(() => {
- const queueOrderNo = this.requestIdToQueueOrderNo[request.id];
- const source = this._getFilePath(queueOrderNo, false);
- const dest = this._getFilePath(queueOrderNo, true);
-
- if (!this.queueOrderNoInProgress[queueOrderNo]) {
- throw new Error(`Cannot mark request ${request.id} as handled, because it is not in progress!`);
- }
-
- if (!request.handledAt) request.handledAt = new Date();
-
- this._updateMetadata(true);
-
- // NOTE: First write to old file and then rename to new one to do the operation atomically.
- // Situation where two files exists at the same time may cause race condition bugs.
- return writeFilePromised(source, JSON.stringify(request, null, 4))
- .then(() => renamePromised(source, dest))
- .then(() => {
- this.pendingCount--;
- this._handledCount++;
- this.inProgressCount--;
- delete this.queueOrderNoInProgress[queueOrderNo];
-
- return {
- requestId: request.id,
- wasAlreadyHandled: false,
- wasAlreadyPresent: true,
- request,
- };
- });
- });
+ await this.initializationPromise;
+ const queueOrderNo = this.requestIdToQueueOrderNo[request.id];
+ const source = this._getFilePath(queueOrderNo, false);
+ const dest = this._getFilePath(queueOrderNo, true);
+
+ if (!this.queueOrderNoInProgress[queueOrderNo]) {
+ throw new Error(`Cannot mark request ${request.id} as handled, because it is not in progress!`);
+ }
+
+ if (!request.handledAt) request.handledAt = new Date();
+
+ this._updateMetadata(true);
+
+ // NOTE: First write to old file and then rename to new one to do the operation atomically.
+ // Situation where two files exists at the same time may cause race condition bugs.
+ await fs.writeFile(source, JSON.stringify(request, null, 4));
+ await fs.rename(source, dest);
+ this.pendingCount--;
+ this._handledCount++;
+ this.inProgressCount--;
+ delete this.queueOrderNoInProgress[queueOrderNo];
+
+ return {
+ requestId: request.id,
+ wasAlreadyHandled: false,
+ wasAlreadyPresent: true,
+ request,
+ };
}
- reclaimRequest(request, opts = {}) {
+ async reclaimRequest(request, opts = {}) {
const { forefront } = validateReclaimRequestParams(request, opts);
- return this.initializationPromise
- .then(() => {
- const oldQueueOrderNo = this.requestIdToQueueOrderNo[request.id];
- const newQueueOrderNo = this._getQueueOrderNo(forefront);
- const source = this._getFilePath(oldQueueOrderNo);
- const dest = this._getFilePath(newQueueOrderNo);
-
- if (!this.queueOrderNoInProgress[oldQueueOrderNo]) {
- throw new Error(`Cannot reclaim request ${request.id}, because it is not in progress!`);
- }
-
- this.requestIdToQueueOrderNo[request.id] = newQueueOrderNo;
-
- this._updateMetadata(true);
-
- // NOTE: First write to old file and then rename to new one to do the operation atomically.
- // Situation where two files exists at the same time may cause race condition bugs.
- return writeFilePromised(source, JSON.stringify(request, null, 4))
- .then(() => renamePromised(source, dest))
- .then(() => {
- this.inProgressCount--;
- delete this.queueOrderNoInProgress[oldQueueOrderNo];
-
- return {
- requestId: request.id,
- wasAlreadyHandled: false,
- wasAlreadyPresent: true,
- request,
- };
- });
- });
+ await this.initializationPromise;
+ const oldQueueOrderNo = this.requestIdToQueueOrderNo[request.id];
+ const newQueueOrderNo = this._getQueueOrderNo(forefront);
+ const source = this._getFilePath(oldQueueOrderNo);
+ const dest = this._getFilePath(newQueueOrderNo);
+
+ if (!this.queueOrderNoInProgress[oldQueueOrderNo]) {
+ throw new Error(`Cannot reclaim request ${request.id}, because it is not in progress!`);
+ }
+
+ this.requestIdToQueueOrderNo[request.id] = newQueueOrderNo;
+
+ this._updateMetadata(true);
+
+ // NOTE: First write to old file and then rename to new one to do the operation atomically.
+ // Situation where two files exists at the same time may cause race condition bugs.
+ await fs.writeFile(source, JSON.stringify(request, null, 4));
+ await fs.rename(source, dest);
+ this.inProgressCount--;
+ delete this.queueOrderNoInProgress[oldQueueOrderNo];
+
+ return {
+ requestId: request.id,
+ wasAlreadyHandled: false,
+ wasAlreadyPresent: true,
+ request,
+ };
}
async isEmpty() {
@@ -1009,7 +998,7 @@ export class RequestQueueLocal {
}
async drop() {
- await emptyDirPromised(this.localStoragePath);
+ await fs.emptyDir(this.localStoragePath);
queuesCache.remove(this.queueId);
}
@@ -1057,13 +1046,10 @@ export class RequestQueueLocal {
*
* @ignore
*/
-const getOrCreateQueue = (queueIdOrName) => {
- return requestQueues.getQueue({ queueId: queueIdOrName })
- .then((existingQueue) => {
- if (existingQueue) return existingQueue;
-
- return requestQueues.getOrCreateQueue({ queueName: queueIdOrName });
- });
+const getOrCreateQueue = async (queueIdOrName) => {
+ const existingQueue = await requestQueues.getQueue({ queueId: queueIdOrName });
+ if (existingQueue) return existingQueue;
+ return requestQueues.getOrCreateQueue({ queueName: queueIdOrName });
};
/**