Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(MemoryStorage): request queues race conditions causing crashes (#1806)

Vlad Frangu083a9db9ebcd
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.json
index 05f55725c..bfa05eec9 100644
--- a/packages/memory-storage/package.json
+++ b/packages/memory-storage/package.json
@@ -51,6 +51,7 @@
"@apify/log": "^2.0.0",
"@crawlee/types": "^3.2.2",
"@crawlee/utils": "^3.2.2",
+ "@sapphire/async-queue": "^1.5.0",
"@sapphire/shapeshift": "^3.0.0",
"content-type": "^1.0.4",
"fs-extra": "^11.0.0",
diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.ts
index c3b75e300..a26608073 100644
--- a/packages/memory-storage/src/fs/request-queue/fs.ts
+++ b/packages/memory-storage/src/fs/request-queue/fs.ts
@@ -1,3 +1,4 @@
+import { AsyncQueue } from '@sapphire/async-queue';
import { ensureDir } from 'fs-extra';
import { readFile, rm } from 'node:fs/promises';
import { dirname, resolve } from 'node:path';
@@ -8,18 +9,29 @@ import type { StorageImplementation } from '../common';
export class RequestQueueFileSystemEntry implements StorageImplementation<InternalRequest> {
private filePath: string;
+ private queue = new AsyncQueue();
constructor(options: CreateStorageImplementationOptions) {
this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);
}
async get() {
- return JSON.parse(await readFile(this.filePath, 'utf-8'));
+ try {
+ await this.queue.wait();
+ return JSON.parse(await readFile(this.filePath, 'utf-8'));
+ } finally {
+ this.queue.shift();
+ }
}
async update(data: InternalRequest) {
- await ensureDir(dirname(this.filePath));
- await lockAndWrite(this.filePath, data);
+ try {
+ await this.queue.wait();
+ await ensureDir(dirname(this.filePath));
+ await lockAndWrite(this.filePath, data);
+ } finally {
+ this.queue.shift();
+ }
}
async delete() {
diff --git a/packages/memory-storage/src/workers/worker-utils.ts b/packages/memory-storage/src/workers/worker-utils.ts
index 5953a01ca..e09c55d5e 100644
--- a/packages/memory-storage/src/workers/worker-utils.ts
+++ b/packages/memory-storage/src/workers/worker-utils.ts
@@ -1,10 +1,11 @@
import log from '@apify/log';
import { ensureDir } from 'fs-extra';
-import { writeFile } from 'node:fs/promises';
+import { writeFile as writeFileP } from 'node:fs/promises';
import { setTimeout } from 'node:timers/promises';
import { resolve } from 'node:path';
import { parentPort } from 'node:worker_threads';
import { lock } from 'proper-lockfile';
+import { writeFile } from 'node:fs';
import type { WorkerReceivedMessage, WorkerUpdateMetadataMessage } from '../utils';
const workerLog = log.child({ prefix: 'MemoryStorageWorker' });
@@ -38,13 +39,21 @@ async function updateMetadata(message: WorkerUpdateMetadataMessage) {
// Write the metadata to the file
const filePath = resolve(dir, '__metadata__.json');
- await writeFile(filePath, JSON.stringify(message.data, null, '\t'));
+ await writeFileP(filePath, JSON.stringify(message.data, null, '\t'));
}
export async function lockAndWrite(filePath: string, data: unknown, stringify = true, retry = 10, timeout = 10): Promise<void> {
try {
const release = await lock(filePath, { realpath: false });
- await writeFile(filePath, stringify ? JSON.stringify(data, null, '\t') : data as Buffer);
+ await new Promise<void>((pResolve, reject) => {
+ writeFile(filePath, stringify ? JSON.stringify(data, null, '\t') : data as Buffer, (err) => {
+ if (err) {
+ reject(err);
+ } else {
+ pResolve();
+ }
+ });
+ });
await release();
} catch (e: any) {
if (e.code === 'ELOCKED' && retry > 0) {
diff --git a/yarn.lock b/yarn.lock
index f48c4ecd9..62676ffde 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -894,6 +894,7 @@ __metadata:
"@apify/log": ^2.0.0
"@crawlee/types": ^3.2.2
"@crawlee/utils": ^3.2.2
+ "@sapphire/async-queue": ^1.5.0
"@sapphire/shapeshift": ^3.0.0
content-type: ^1.0.4
fs-extra: ^11.0.0
@@ -1947,6 +1948,13 @@ __metadata:
languageName: node
linkType: hard
+"@sapphire/async-queue@npm:^1.5.0":
+ version: 1.5.0
+ resolution: "@sapphire/async-queue@npm:1.5.0"
+ checksum: 983dbd1fd1b1798496e5edb6a0db7e4d90015160e1028f20475eab0a92625513f1e8d938bc0305811a9cec461c94e01b1e4191615ff03ba49356f568f3255250
+ languageName: node
+ linkType: hard
+
"@sapphire/shapeshift@npm:^3.0.0, @sapphire/shapeshift@npm:^3.6.0":
version: 3.8.1
resolution: "@sapphire/shapeshift@npm:3.8.1"