Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(MemoryStorage): lock request JSON file when reading to support multiple process crawling (#2215)
Vlad Frangueb84ce9ce554
diff --git a/packages/memory-storage/src/background-handler/fs-utils.ts b/packages/memory-storage/src/background-handler/fs-utils.tsindex d446189de..91368b636 100644--- a/packages/memory-storage/src/background-handler/fs-utils.ts+++ b/packages/memory-storage/src/background-handler/fs-utils.ts@@ -39,8 +39,7 @@ async function updateMetadata(message: BackgroundHandlerUpdateMetadataMessage) {}export async function lockAndWrite(filePath: string, data: unknown, stringify = true, retry = 10, timeout = 10): Promise<void> {- try {- const release = await lock(filePath, { realpath: false });+ await lockAndCallback(filePath, async () => {await new Promise<void>((pResolve, reject) => {writeFile(filePath, stringify ? JSON.stringify(data, null, '\t') : data as Buffer, (err) => {if (err) {@@ -50,11 +49,30 @@ export async function lockAndWrite(filePath: string, data: unknown, stringify =}});});- await release();+ }, retry, timeout);+}++export async function lockAndCallback<Callback extends () => Promise<any>>(+ filePath: string,+ callback: Callback,+ retry = 10,+ timeout = 10,+): Promise<Awaited<ReturnType<Callback>>> {+ let release: (() => Promise<void>) | null = null;+ try {+ release = await lock(filePath, { realpath: false });++ return await callback();} catch (e: any) {if (e.code === 'ELOCKED' && retry > 0) {await setTimeout(timeout);- return lockAndWrite(filePath, data, stringify, retry - 1, timeout * 2);+ return lockAndCallback(filePath, callback, retry - 1, timeout * 2);+ }++ throw e;+ } finally {+ if (release) {+ await release();}}}diff --git a/packages/memory-storage/src/fs/common.ts b/packages/memory-storage/src/fs/common.tsindex 13894caf0..1a26019fe 100644--- a/packages/memory-storage/src/fs/common.ts+++ b/packages/memory-storage/src/fs/common.ts@@ -1,5 +1,5 @@export interface StorageImplementation<T> {- get(): Promise<T>;+ get(force?: boolean): Promise<T>;update(data: T): void | Promise<void>;delete(): void | Promise<void>;}diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.tsindex cfc94eb50..a4937e10e 100644--- a/packages/memory-storage/src/fs/request-queue/fs.ts+++ b/packages/memory-storage/src/fs/request-queue/fs.ts@@ -4,7 +4,7 @@ import { dirname, resolve } from 'node:path';import { AsyncQueue } from '@sapphire/async-queue';import { ensureDir } from 'fs-extra';-import { lockAndWrite } from '../../background-handler/fs-utils';+import { lockAndCallback, lockAndWrite } from '../../background-handler/fs-utils';import type { InternalRequest } from '../../resource-clients/request-queue';import type { StorageImplementation } from '../common';@@ -14,6 +14,7 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Internprivate filePath: string;private fsQueue = new AsyncQueue();private data?: InternalRequest;+ private directoryExists = false;/*** A "sweep" timeout that is created/refreshed whenever this entry is accessed/updated.@@ -25,20 +26,22 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Internthis.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);}- async get() {+ async get(force = false) {await this.fsQueue.wait();this.setOrRefreshSweepTimeout();- if (this.data) {+ if (this.data && !force) {this.fsQueue.shift();return this.data;}try {- const req = JSON.parse(await readFile(this.filePath, 'utf-8'));- this.data = req;+ return await lockAndCallback(this.filePath, async () => {+ const req = JSON.parse(await readFile(this.filePath, 'utf-8'));+ this.data = req;- return req;+ return req;+ });} finally {this.fsQueue.shift();}@@ -49,7 +52,11 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Internthis.data = data;try {- await ensureDir(dirname(this.filePath));+ if (!this.directoryExists) {+ await ensureDir(dirname(this.filePath));+ this.directoryExists = true;+ }+await lockAndWrite(this.filePath, data);} finally {this.setOrRefreshSweepTimeout();diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.tsindex 2e2d0589a..412ec4441 100644--- a/packages/memory-storage/src/resource-clients/request-queue.ts+++ b/packages/memory-storage/src/resource-clients/request-queue.ts@@ -200,7 +200,8 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuebreak;}- const request = await storageEntry.get();+ // Always fetch from fs, as this also locks and we do not want to end up in a state where another process locked the request but we have cached it as unlocked+ const request = await storageEntry.get(true);if (isLocked(request)) {continue;