Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: queue up all fs actions in memory storage (#1866)
Vlad Frangu8ea7a4f3622e
diff --git a/packages/memory-storage/src/fs/dataset/fs.ts b/packages/memory-storage/src/fs/dataset/fs.tsindex a280073d2..4d4b2a6b2 100644--- a/packages/memory-storage/src/fs/dataset/fs.ts+++ b/packages/memory-storage/src/fs/dataset/fs.ts@@ -1,27 +1,41 @@import { readFile, rm } from 'node:fs/promises';import { dirname, resolve } from 'node:path';import { ensureDir } from 'fs-extra';+import { AsyncQueue } from '@sapphire/async-queue';import type { CreateStorageImplementationOptions } from './index';import type { StorageImplementation } from '../common';import { lockAndWrite } from '../../workers/worker-utils';export class DatasetFileSystemEntry<Data> implements StorageImplementation<Data> {private filePath: string;+ private fsQueue = new AsyncQueue();constructor(options: CreateStorageImplementationOptions) {this.filePath = resolve(options.storeDirectory, `${options.entityId}.json`);}async get() {- return JSON.parse(await readFile(this.filePath, 'utf-8'));+ await this.fsQueue.wait();+ try {+ return JSON.parse(await readFile(this.filePath, 'utf-8'));+ } finally {+ this.fsQueue.shift();+ }}async update(data: Data) {- await ensureDir(dirname(this.filePath));- await lockAndWrite(this.filePath, data);+ await this.fsQueue.wait();+ try {+ await ensureDir(dirname(this.filePath));+ await lockAndWrite(this.filePath, data);+ } finally {+ this.fsQueue.shift();+ }}async delete() {+ await this.fsQueue.wait();await rm(this.filePath, { force: true });+ this.fsQueue.shift();}}diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.tsindex af87497fb..c3b9abc69 100644--- a/packages/memory-storage/src/fs/key-value-store/fs.ts+++ b/packages/memory-storage/src/fs/key-value-store/fs.ts@@ -2,6 +2,7 @@ import { ensureDir } from 'fs-extra';import { readFile, rm } from 'node:fs/promises';import { dirname, resolve } from 'node:path';import { basename } from 'node:path/win32';+import { AsyncQueue } from '@sapphire/async-queue';import type { CreateStorageImplementationOptions } from '.';import type { InternalKeyRecord } from '../../resource-clients/key-value-store';import { memoryStorageLog } from '../../utils';@@ -15,6 +16,7 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKeprivate filePath!: string;private fileMetadataPath!: string;private rawRecord!: Omit<InternalKeyRecord, 'value'>;+ private fsQueue = new AsyncQueue();constructor(options: CreateStorageImplementationOptions) {this.storeDirectory = options.storeDirectory;@@ -22,6 +24,7 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe}async get(): Promise<InternalKeyRecord> {+ await this.fsQueue.wait();let file: Buffer | string;try {@@ -39,6 +42,8 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe// This is impossible to happen, but just in casethrow new Error(`Could not find file at ${this.filePath}`);}+ } finally {+ this.fsQueue.shift();}return {@@ -48,22 +53,29 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe}async update(data: InternalKeyRecord) {+ await this.fsQueue.wait();this.filePath ??= resolve(this.storeDirectory, `${data.key}.${data.extension}`);this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);const { value, ...rest } = data;this.rawRecord = rest;- await ensureDir(dirname(this.filePath));- await lockAndWrite(this.filePath, value, false);+ try {+ await ensureDir(dirname(this.filePath));+ await lockAndWrite(this.filePath, value, false);- if (this.writeMetadata) {- await lockAndWrite(this.fileMetadataPath, JSON.stringify(rest), true);+ if (this.writeMetadata) {+ await lockAndWrite(this.fileMetadataPath, JSON.stringify(rest), true);+ }+ } finally {+ this.fsQueue.shift();}}async delete() {+ await this.fsQueue.wait();await rm(this.filePath, { force: true });await rm(this.fileMetadataPath, { force: true });+ this.fsQueue.shift();}}diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.tsindex a26608073..668d3350f 100644--- a/packages/memory-storage/src/fs/request-queue/fs.ts+++ b/packages/memory-storage/src/fs/request-queue/fs.ts@@ -9,32 +9,34 @@ import type { StorageImplementation } from '../common';export class RequestQueueFileSystemEntry implements StorageImplementation<InternalRequest> {private filePath: string;- private queue = new AsyncQueue();+ private fsQueue = new AsyncQueue();constructor(options: CreateStorageImplementationOptions) {this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);}async get() {+ await this.fsQueue.wait();try {- await this.queue.wait();return JSON.parse(await readFile(this.filePath, 'utf-8'));} finally {- this.queue.shift();+ this.fsQueue.shift();}}async update(data: InternalRequest) {+ await this.fsQueue.wait();try {- await this.queue.wait();await ensureDir(dirname(this.filePath));await lockAndWrite(this.filePath, data);} finally {- this.queue.shift();+ this.fsQueue.shift();}}async delete() {+ await this.fsQueue.wait();await rm(this.filePath, { force: true });+ this.fsQueue.shift();}}