Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(MemoryStorage): request queues race conditions causing crashes (#1806)
Vlad Frangu083a9db9ebcd
diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 05f55725c..bfa05eec9 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -51,6 +51,7 @@"@apify/log": "^2.0.0","@crawlee/types": "^3.2.2","@crawlee/utils": "^3.2.2",+ "@sapphire/async-queue": "^1.5.0","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4","fs-extra": "^11.0.0",diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.tsindex c3b75e300..a26608073 100644--- a/packages/memory-storage/src/fs/request-queue/fs.ts+++ b/packages/memory-storage/src/fs/request-queue/fs.ts@@ -1,3 +1,4 @@+import { AsyncQueue } from '@sapphire/async-queue';import { ensureDir } from 'fs-extra';import { readFile, rm } from 'node:fs/promises';import { dirname, resolve } from 'node:path';@@ -8,18 +9,29 @@ import type { StorageImplementation } from '../common';export class RequestQueueFileSystemEntry implements StorageImplementation<InternalRequest> {private filePath: string;+ private queue = new AsyncQueue();constructor(options: CreateStorageImplementationOptions) {this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);}async get() {- return JSON.parse(await readFile(this.filePath, 'utf-8'));+ try {+ await this.queue.wait();+ return JSON.parse(await readFile(this.filePath, 'utf-8'));+ } finally {+ this.queue.shift();+ }}async update(data: InternalRequest) {- await ensureDir(dirname(this.filePath));- await lockAndWrite(this.filePath, data);+ try {+ await this.queue.wait();+ await ensureDir(dirname(this.filePath));+ await lockAndWrite(this.filePath, data);+ } finally {+ this.queue.shift();+ }}async delete() {diff --git a/packages/memory-storage/src/workers/worker-utils.ts b/packages/memory-storage/src/workers/worker-utils.tsindex 5953a01ca..e09c55d5e 100644--- a/packages/memory-storage/src/workers/worker-utils.ts+++ b/packages/memory-storage/src/workers/worker-utils.ts@@ -1,10 +1,11 @@import log from '@apify/log';import { ensureDir } from 'fs-extra';-import { writeFile } from 'node:fs/promises';+import { writeFile as writeFileP } from 'node:fs/promises';import { setTimeout } from 'node:timers/promises';import { resolve } from 'node:path';import { parentPort } from 'node:worker_threads';import { lock } from 'proper-lockfile';+import { writeFile } from 'node:fs';import type { WorkerReceivedMessage, WorkerUpdateMetadataMessage } from '../utils';const workerLog = log.child({ prefix: 'MemoryStorageWorker' });@@ -38,13 +39,21 @@ async function updateMetadata(message: WorkerUpdateMetadataMessage) {// Write the metadata to the fileconst filePath = resolve(dir, '__metadata__.json');- await writeFile(filePath, JSON.stringify(message.data, null, '\t'));+ await writeFileP(filePath, JSON.stringify(message.data, null, '\t'));}export async function lockAndWrite(filePath: string, data: unknown, stringify = true, retry = 10, timeout = 10): Promise<void> {try {const release = await lock(filePath, { realpath: false });- await writeFile(filePath, stringify ? JSON.stringify(data, null, '\t') : data as Buffer);+ await new Promise<void>((pResolve, reject) => {+ writeFile(filePath, stringify ? JSON.stringify(data, null, '\t') : data as Buffer, (err) => {+ if (err) {+ reject(err);+ } else {+ pResolve();+ }+ });+ });await release();} catch (e: any) {if (e.code === 'ELOCKED' && retry > 0) {diff --git a/yarn.lock b/yarn.lockindex f48c4ecd9..62676ffde 100644--- a/yarn.lock+++ b/yarn.lock@@ -894,6 +894,7 @@ __metadata:"@apify/log": ^2.0.0"@crawlee/types": ^3.2.2"@crawlee/utils": ^3.2.2+ "@sapphire/async-queue": ^1.5.0"@sapphire/shapeshift": ^3.0.0content-type: ^1.0.4fs-extra: ^11.0.0@@ -1947,6 +1948,13 @@ __metadata:languageName: nodelinkType: hard+"@sapphire/async-queue@npm:^1.5.0":+ version: 1.5.0+ resolution: "@sapphire/async-queue@npm:1.5.0"+ checksum: 983dbd1fd1b1798496e5edb6a0db7e4d90015160e1028f20475eab0a92625513f1e8d938bc0305811a9cec461c94e01b1e4191615ff03ba49356f568f3255250+ languageName: node+ linkType: hard+"@sapphire/shapeshift@npm:^3.0.0, @sapphire/shapeshift@npm:^3.6.0":version: 3.8.1resolution: "@sapphire/shapeshift@npm:3.8.1"