Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(MemoryStorage): read from fs if persistStorage is enabled, ram only otherwise (#1761)
Vlad Frangue9039809a0c0
diff --git a/packages/memory-storage/src/cache-helpers.ts b/packages/memory-storage/src/cache-helpers.tsindex 2860b7e6d..ac9aed857 100644--- a/packages/memory-storage/src/cache-helpers.ts+++ b/packages/memory-storage/src/cache-helpers.ts@@ -2,9 +2,12 @@ import type * as storage from '@crawlee/types';import { access, opendir, readFile } from 'node:fs/promises';import { extname, resolve } from 'node:path';import mimeTypes from 'mime-types';+import json5 from 'json5';import type { InternalKeyRecord } from './resource-clients/key-value-store';-import type { InternalRequest } from './resource-clients/request-queue';import type { MemoryStorage } from './memory-storage';+import { DatasetFileSystemEntry } from './fs/dataset/fs';+import { KeyValueFileSystemEntry } from './fs/key-value-store/fs';+import { RequestQueueFileSystemEntry } from './fs/request-queue/fs';import { memoryStorageLog } from './utils';const uuidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;@@ -32,7 +35,9 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entrlet id: string | undefined;let name: string | undefined;let itemCount = 0;- const entries = new Map<string, storage.Dictionary>();++ const entries = new Set<string>();+let createdAt = new Date();let accessedAt = new Date();let modifiedAt = new Date();@@ -59,13 +64,8 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entrcontinue;}- const fileContent = await readFile(resolve(datasetDir, entry.name), 'utf8');- if (!fileContent) continue;-- const entryContent = JSON.parse(fileContent) as storage.Dictionary;const entryName = entry.name.split('.')[0];-- entries.set(entryName, entryContent);+ entries.add(entryName);if (!hasSeenMetadataFile) {itemCount++;@@ -96,9 +96,12 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entrnewClient.modifiedAt = modifiedAt;newClient.itemCount = itemCount;- for (const [entryId, content] of entries) {+ for (const entryId of entries.values()) {+ // We create a file system entry instead of possibly making an in-memory one to allow the pre-included data to be used on demand+ const entry = new DatasetFileSystemEntry({ storeDirectory: datasetDir, entityId: entryId, persistStorage: true });+// eslint-disable-next-line dot-notation- newClient['datasetEntries'].set(entryId, { ...content });+ newClient['datasetEntries'].set(entryId, entry);}client.datasetClientsHandled.push(newClient);@@ -131,7 +134,10 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStoragelet createdAt = new Date();let accessedAt = new Date();let modifiedAt = new Date();- const internalRecords = new Map<string, InternalKeyRecord>();++ type FsRecord = Omit<InternalKeyRecord, 'value'>;+ const internalRecords = new Map<string, FsRecord>();+ let hasSeenMetadataForEntry = false;for await (const entry of directoryEntries) {if (entry.isFile()) {@@ -151,49 +157,42 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage}if (entry.name.includes('.__metadata__.')) {+ hasSeenMetadataForEntry = true;+// This is an entry's metadata file, we can use it to create/extend the recordconst fileContent = await readFile(resolve(keyValueStoreDir, entry.name), 'utf8');if (!fileContent) continue;- const metadata = JSON.parse(fileContent) as Omit<InternalKeyRecord, 'value'>;+ const metadata = JSON.parse(fileContent) as FsRecord;const newRecord = {...internalRecords.get(metadata.key),...metadata,- } as InternalKeyRecord;+ } as FsRecord;internalRecords.set(metadata.key, newRecord);continue;}+ // This is an entry in the store, we can use it to create/extend the recordconst fileContent = await readFile(resolve(keyValueStoreDir, entry.name));const fileExtension = extname(entry.name);const contentType = mimeTypes.contentType(entry.name) || 'text/plain';const extension = mimeTypes.extension(contentType) as string;- let finalFileContent: Buffer | string = fileContent;-- if (!fileExtension) {- memoryStorageLog.warning([- `Key-value entry "${entry.name}" for store ${entryNameOrId} does not have a file extension, assuming it as text.`,- 'If you want to have correct interpretation of the file, you should add a file extension to the entry.',- ].join('\n'));- finalFileContent = fileContent.toString('utf8');- } else if (contentType.includes('application/json')) {+ // This is kept for backwards compatibility / to ignore invalid JSON files+ if (contentType.includes('application/json')) {const stringifiedJson = fileContent.toString('utf8');+try {- // Try parsing the JSON ahead of time (not ideal but solves invalid files being loaded into stores)- JSON.parse(stringifiedJson);- finalFileContent = stringifiedJson;+ json5.parse(stringifiedJson);} catch {memoryStorageLog.warning(`Key-value entry "${entry.name}" for store ${entryNameOrId} has invalid JSON content and will be ignored from the store.`,);continue;}- } else if (contentType.includes('text/plain')) {- finalFileContent = fileContent.toString('utf8');}const nameSplit = entry.name.split('.');@@ -204,13 +203,12 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorageconst key = nameSplit.join('.');- const newRecord: InternalKeyRecord = {+ const newRecord = {key,extension,- value: finalFileContent,contentType,...internalRecords.get(key),- };+ } satisfies FsRecord;internalRecords.set(key, newRecord);}@@ -239,8 +237,18 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStoragenewClient.modifiedAt = modifiedAt;for (const [key, record] of internalRecords) {+ // We create a file system entry instead of possibly making an in-memory one to allow the pre-included data to be used on demand+ const entry = new KeyValueFileSystemEntry({ persistStorage: true, storeDirectory: keyValueStoreDir, writeMetadata: hasSeenMetadataForEntry });++ // eslint-disable-next-line dot-notation+ entry['rawRecord'] = { ...record };+ // eslint-disable-next-line dot-notation+ entry['filePath'] = resolve(keyValueStoreDir, `${record.key}.${record.extension}`);+ // eslint-disable-next-line dot-notation+ entry['fileMetadataPath'] = resolve(keyValueStoreDir, `${record.key}.__metadata__.json`);+// eslint-disable-next-line dot-notation- newClient['keyValueEntries'].set(key, { ...record });+ newClient['keyValueEntries'].set(key, entry);}client.keyValueStoresHandled.push(newClient);@@ -275,7 +283,7 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryNlet modifiedAt = new Date();let pendingRequestCount = 0;let handledRequestCount = 0;- const entries: InternalRequest[] = [];+ const entries = new Set<string>();for await (const entry of directoryEntries) {if (entry.isFile()) {@@ -298,11 +306,8 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryNbreak;}default: {- const fileContent = await readFile(resolve(requestQueueDir, entry.name), 'utf8');- if (!fileContent) continue;-- const request = JSON.parse(fileContent) as InternalRequest;- entries.push(request);+ const entryName = entry.name.split('.')[0];+ entries.add(entryName);}}}@@ -332,9 +337,15 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryNnewClient.pendingRequestCount = pendingRequestCount;newClient.handledRequestCount = handledRequestCount;- for (const entry of entries) {+ for (const requestId of entries) {+ const entry = new RequestQueueFileSystemEntry({+ persistStorage: true,+ requestId,+ storeDirectory: requestQueueDir,+ });+// eslint-disable-next-line dot-notation- newClient['requests'].set(entry.id, entry);+ newClient['requests'].set(requestId, entry);}client.requestQueuesHandled.push(newClient);diff --git a/packages/memory-storage/src/fs/common.ts b/packages/memory-storage/src/fs/common.tsnew file mode 100644index 000000000..13894caf0--- /dev/null+++ b/packages/memory-storage/src/fs/common.ts@@ -0,0 +1,5 @@+export interface StorageImplementation<T> {+ get(): Promise<T>;+ update(data: T): void | Promise<void>;+ delete(): void | Promise<void>;+}diff --git a/packages/memory-storage/src/fs/dataset/fs.ts b/packages/memory-storage/src/fs/dataset/fs.tsnew file mode 100644index 000000000..a280073d2--- /dev/null+++ b/packages/memory-storage/src/fs/dataset/fs.ts@@ -0,0 +1,27 @@+import { readFile, rm } from 'node:fs/promises';+import { dirname, resolve } from 'node:path';+import { ensureDir } from 'fs-extra';+import type { CreateStorageImplementationOptions } from './index';+import type { StorageImplementation } from '../common';+import { lockAndWrite } from '../../workers/worker-utils';++export class DatasetFileSystemEntry<Data> implements StorageImplementation<Data> {+ private filePath: string;++ constructor(options: CreateStorageImplementationOptions) {+ this.filePath = resolve(options.storeDirectory, `${options.entityId}.json`);+ }++ async get() {+ return JSON.parse(await readFile(this.filePath, 'utf-8'));+ }++ async update(data: Data) {+ await ensureDir(dirname(this.filePath));+ await lockAndWrite(this.filePath, data);+ }++ async delete() {+ await rm(this.filePath, { force: true });+ }+}diff --git a/packages/memory-storage/src/fs/dataset/index.ts b/packages/memory-storage/src/fs/dataset/index.tsnew file mode 100644index 000000000..1f5aaa94d--- /dev/null+++ b/packages/memory-storage/src/fs/dataset/index.ts@@ -0,0 +1,19 @@+import type { Dictionary } from '@crawlee/types';+import type { StorageImplementation } from '../common';+import { DatasetFileSystemEntry } from './fs';+import { DatasetMemoryEntry } from './memory';++export function createDatasetStorageImplementation<Data extends Dictionary>(options: CreateStorageImplementationOptions): StorageImplementation<Data> {+ if (options.persistStorage) {+ return new DatasetFileSystemEntry<Data>(options);+ }++ return new DatasetMemoryEntry<Data>();+}++export interface CreateStorageImplementationOptions {+ persistStorage: boolean;+ storeDirectory: string;+ /** The actual id of the file to save */+ entityId: string;+}diff --git a/packages/memory-storage/src/fs/dataset/memory.ts b/packages/memory-storage/src/fs/dataset/memory.tsnew file mode 100644index 000000000..569b77beb--- /dev/null+++ b/packages/memory-storage/src/fs/dataset/memory.ts@@ -0,0 +1,17 @@+import type { StorageImplementation } from '../common';++export class DatasetMemoryEntry<Data> implements StorageImplementation<Data> {+ private data!: Data;++ async get() {+ return this.data;+ }++ update(data: Data) {+ this.data = data;+ }++ delete() {+ // No-op+ }+}diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.tsnew file mode 100644index 000000000..af87497fb--- /dev/null+++ b/packages/memory-storage/src/fs/key-value-store/fs.ts@@ -0,0 +1,69 @@+import { ensureDir } from 'fs-extra';+import { readFile, rm } from 'node:fs/promises';+import { dirname, resolve } from 'node:path';+import { basename } from 'node:path/win32';+import type { CreateStorageImplementationOptions } from '.';+import type { InternalKeyRecord } from '../../resource-clients/key-value-store';+import { memoryStorageLog } from '../../utils';+import { lockAndWrite } from '../../workers/worker-utils';+import type { StorageImplementation } from '../common';++export class KeyValueFileSystemEntry implements StorageImplementation<InternalKeyRecord> {+ private storeDirectory: string;+ private writeMetadata: boolean;++ private filePath!: string;+ private fileMetadataPath!: string;+ private rawRecord!: Omit<InternalKeyRecord, 'value'>;++ constructor(options: CreateStorageImplementationOptions) {+ this.storeDirectory = options.storeDirectory;+ this.writeMetadata = options.writeMetadata;+ }++ async get(): Promise<InternalKeyRecord> {+ let file: Buffer | string;++ try {+ file = await readFile(this.filePath);+ } catch {+ try {+ // Try without extension+ file = await readFile(resolve(this.storeDirectory, this.rawRecord.key));+ memoryStorageLog.warning([+ `Key-value entry "${this.rawRecord.key}" for store ${basename(this.storeDirectory)} does not have a file extension, assuming it as text.`,+ 'If you want to have correct interpretation of the file, you should add a file extension to the entry.',+ ].join('\n'));+ file = file.toString('utf-8');+ } catch {+ // This is impossible to happen, but just in case+ throw new Error(`Could not find file at ${this.filePath}`);+ }+ }++ return {+ ...this.rawRecord,+ value: file,+ };+ }++ async update(data: InternalKeyRecord) {+ this.filePath ??= resolve(this.storeDirectory, `${data.key}.${data.extension}`);+ this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);++ const { value, ...rest } = data;+ this.rawRecord = rest;++ await ensureDir(dirname(this.filePath));+ await lockAndWrite(this.filePath, value, false);++ if (this.writeMetadata) {+ await lockAndWrite(this.fileMetadataPath, JSON.stringify(rest), true);+ }+ }++ async delete() {+ await rm(this.filePath, { force: true });+ await rm(this.fileMetadataPath, { force: true });+ }+}diff --git a/packages/memory-storage/src/fs/key-value-store/index.ts b/packages/memory-storage/src/fs/key-value-store/index.tsnew file mode 100644index 000000000..5ca6b0394--- /dev/null+++ b/packages/memory-storage/src/fs/key-value-store/index.ts@@ -0,0 +1,18 @@+import type { InternalKeyRecord } from '../../resource-clients/key-value-store';+import type { StorageImplementation } from '../common';+import { KeyValueFileSystemEntry } from './fs';+import { KeyValueMemoryEntry } from './memory';++export function createKeyValueStorageImplementation(options: CreateStorageImplementationOptions): StorageImplementation<InternalKeyRecord> {+ if (options.persistStorage) {+ return new KeyValueFileSystemEntry(options);+ }++ return new KeyValueMemoryEntry();+}++export interface CreateStorageImplementationOptions {+ persistStorage: boolean;+ storeDirectory: string;+ writeMetadata: boolean;+}diff --git a/packages/memory-storage/src/fs/key-value-store/memory.ts b/packages/memory-storage/src/fs/key-value-store/memory.tsnew file mode 100644index 000000000..bc9e9e4a0--- /dev/null+++ b/packages/memory-storage/src/fs/key-value-store/memory.ts@@ -0,0 +1,18 @@+import type { InternalKeyRecord } from '../../resource-clients/key-value-store';+import type { StorageImplementation } from '../common';++export class KeyValueMemoryEntry implements StorageImplementation<InternalKeyRecord> {+ private data!: InternalKeyRecord;++ async get() {+ return this.data;+ }++ update(data: InternalKeyRecord) {+ this.data = data;+ }++ delete() {+ // No-op+ }+}diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.tsnew file mode 100644index 000000000..c3b75e300--- /dev/null+++ b/packages/memory-storage/src/fs/request-queue/fs.ts@@ -0,0 +1,28 @@+import { ensureDir } from 'fs-extra';+import { readFile, rm } from 'node:fs/promises';+import { dirname, resolve } from 'node:path';+import type { CreateStorageImplementationOptions } from '.';+import type { InternalRequest } from '../../resource-clients/request-queue';+import { lockAndWrite } from '../../workers/worker-utils';+import type { StorageImplementation } from '../common';++export class RequestQueueFileSystemEntry implements StorageImplementation<InternalRequest> {+ private filePath: string;++ constructor(options: CreateStorageImplementationOptions) {+ this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);+ }++ async get() {+ return JSON.parse(await readFile(this.filePath, 'utf-8'));+ }++ async update(data: InternalRequest) {+ await ensureDir(dirname(this.filePath));+ await lockAndWrite(this.filePath, data);+ }++ async delete() {+ await rm(this.filePath, { force: true });+ }+}diff --git a/packages/memory-storage/src/fs/request-queue/index.ts b/packages/memory-storage/src/fs/request-queue/index.tsnew file mode 100644index 000000000..b38bffaf9--- /dev/null+++ b/packages/memory-storage/src/fs/request-queue/index.ts@@ -0,0 +1,18 @@+import type { InternalRequest } from '../../resource-clients/request-queue';+import type { StorageImplementation } from '../common';+import { RequestQueueFileSystemEntry } from './fs';+import { RequestQueueMemoryEntry } from './memory';++export function createRequestQueueStorageImplementation(options: CreateStorageImplementationOptions): StorageImplementation<InternalRequest> {+ if (options.persistStorage) {+ return new RequestQueueFileSystemEntry(options);+ }++ return new RequestQueueMemoryEntry();+}++export interface CreateStorageImplementationOptions {+ persistStorage: boolean;+ storeDirectory: string;+ requestId: string;+}diff --git a/packages/memory-storage/src/fs/request-queue/memory.ts b/packages/memory-storage/src/fs/request-queue/memory.tsnew file mode 100644index 000000000..fb4e982bd--- /dev/null+++ b/packages/memory-storage/src/fs/request-queue/memory.ts@@ -0,0 +1,18 @@+import type { InternalRequest } from '../../resource-clients/request-queue';+import type { StorageImplementation } from '../common';++export class RequestQueueMemoryEntry implements StorageImplementation<InternalRequest> {+ private data!: InternalRequest;++ async get() {+ return this.data;+ }++ update(data: InternalRequest) {+ this.data = data;+ }++ delete() {+ // No-op+ }+}diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.tsindex d0dd3494d..00fd317db 100644--- a/packages/memory-storage/src/resource-clients/dataset.ts+++ b/packages/memory-storage/src/resource-clients/dataset.ts@@ -11,6 +11,8 @@ import { StorageTypes } from '../consts';import { BaseClient } from './common/base-client';import { sendWorkerMessage } from '../workers/instance';import { findOrCacheDatasetByPossibleId } from '../cache-helpers';+import type { StorageImplementation } from '../fs/common';+import { createDatasetStorageImplementation } from '../fs/dataset';/*** This is what API returns in the x-apify-pagination-limit@@ -39,7 +41,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseCliitemCount = 0;datasetDirectory: string;- private readonly datasetEntries = new Map<string, Data>();+ private readonly datasetEntries = new Map<string, StorageImplementation<Data>>();private readonly client: MemoryStorage;constructor(options: DatasetClientOptions) {@@ -141,7 +143,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseClifor (let idx = start; idx < end; idx++) {const entryNumber = this.generateLocalEntryName(idx);- items.push(existingStoreById.datasetEntries.get(entryNumber)!);+ items.push(await existingStoreById.datasetEntries.get(entryNumber)!.get());}existingStoreById.updateTimestamps(false);@@ -176,23 +178,19 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseClifor (const entry of normalized) {const idx = this.generateLocalEntryName(++existingStoreById.itemCount);+ const storageEntry = createDatasetStorageImplementation({+ entityId: idx,+ persistStorage: this.client.persistStorage,+ storeDirectory: existingStoreById.datasetDirectory,+ });- existingStoreById.datasetEntries.set(idx, entry);+ await storageEntry.update(entry);++ existingStoreById.datasetEntries.set(idx, storageEntry);addedIds.push(idx);}- const dataEntries: [string, Dictionary][] = addedIds.map((id) => [id, existingStoreById.datasetEntries.get(id)!]);-existingStoreById.updateTimestamps(true);- sendWorkerMessage({- data: dataEntries,- action: 'update-entries',- entityType: 'datasets',- entityDirectory: existingStoreById.datasetDirectory,- id: existingStoreById.name ?? existingStoreById.id,- writeMetadata: this.client.writeMetadata,- persistStorage: this.client.persistStorage,- });}toDatasetInfo(): storage.DatasetInfo {diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.tsindex 5b3ae1699..3e7eec4b2 100644--- a/packages/memory-storage/src/resource-clients/key-value-store.ts+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts@@ -13,6 +13,8 @@ import { isBuffer, isStream } from '../utils';import { BaseClient } from './common/base-client';import { sendWorkerMessage } from '../workers/instance';import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers';+import type { StorageImplementation } from '../fs/common';+import { createKeyValueStorageImplementation } from '../fs/key-value-store';const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';@@ -37,7 +39,7 @@ export class KeyValueStoreClient extends BaseClient {modifiedAt = new Date();keyValueStoreDirectory: string;- private readonly keyValueEntries = new Map<string, InternalKeyRecord>();+ private readonly keyValueEntries = new Map<string, StorageImplementation<InternalKeyRecord>>();private readonly client: MemoryStorage;constructor(options: KeyValueStoreClientOptions) {@@ -125,7 +127,9 @@ export class KeyValueStoreClient extends BaseClient {const items = [];- for (const record of existingStoreById.keyValueEntries.values()) {+ for (const storageEntry of existingStoreById.keyValueEntries.values()) {+ const record = await storageEntry.get();+const size = Buffer.byteLength(record.value);items.push({key: record.key,@@ -183,12 +187,14 @@ export class KeyValueStoreClient extends BaseClient {this.throwOnNonExisting(StorageTypes.KeyValueStore);}- const entry = existingStoreById.keyValueEntries.get(key);+ const storageEntry = existingStoreById.keyValueEntries.get(key);- if (!entry) {+ if (!storageEntry) {return undefined;}+ const entry = await storageEntry.get();+const record: storage.KeyValueStoreRecord = {key: entry.key,value: entry.value,@@ -256,28 +262,24 @@ export class KeyValueStoreClient extends BaseClient {value = Buffer.concat(chunks);}- const entry = {+ const _record = {extension,key,value,contentType,- };+ } satisfies InternalKeyRecord;++ const entry = createKeyValueStorageImplementation({+ persistStorage: this.client.persistStorage,+ storeDirectory: existingStoreById.keyValueStoreDirectory,+ writeMetadata: existingStoreById.client.writeMetadata,+ });++ await entry.update(_record);existingStoreById.keyValueEntries.set(key, entry);existingStoreById.updateTimestamps(true);- sendWorkerMessage({- action: 'update-entries',- data: {- action: 'set',- record: entry,- },- entityType: 'keyValueStores',- entityDirectory: existingStoreById.keyValueStoreDirectory,- id: existingStoreById.name ?? existingStoreById.id,- writeMetadata: this.client.writeMetadata,- persistStorage: this.client.persistStorage,- });}async deleteRecord(key: string): Promise<void> {@@ -295,18 +297,7 @@ export class KeyValueStoreClient extends BaseClient {if (entry) {existingStoreById.keyValueEntries.delete(key);existingStoreById.updateTimestamps(true);- sendWorkerMessage({- action: 'update-entries',- data: {- action: 'delete',- record: entry,- },- entityType: 'keyValueStores',- entityDirectory: existingStoreById.keyValueStoreDirectory,- id: existingStoreById.name ?? existingStoreById.id,- writeMetadata: this.client.writeMetadata,- persistStorage: this.client.persistStorage,- });+ await entry.delete();}}diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.tsindex 400039f4e..58d263528 100644--- a/packages/memory-storage/src/resource-clients/request-queue.ts+++ b/packages/memory-storage/src/resource-clients/request-queue.ts@@ -10,6 +10,8 @@ import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils';import { BaseClient } from './common/base-client';import { sendWorkerMessage } from '../workers/instance';import { findRequestQueueByPossibleId } from '../cache-helpers';+import type { StorageImplementation } from '../fs/common';+import { createRequestQueueStorageImplementation } from '../fs/request-queue';const requestShape = s.object({id: s.string,@@ -54,7 +56,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuependingRequestCount = 0;requestQueueDirectory: string;- private readonly requests = new Map<string, InternalRequest>();+ private readonly requests = new Map<string, StorageImplementation<InternalRequest>>();private readonly client: MemoryStorage;constructor(options: RequestQueueClientOptions) {@@ -141,11 +143,13 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueconst items = [];- for (const request of existingQueueById.requests.values()) {+ for (const storageEntry of existingQueueById.requests.values()) {if (items.length === limit) {break;}+ const request = await storageEntry.get();+if (request.orderNo) {items.push(request);}@@ -171,10 +175,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueconst requestModel = this._createInternalRequest(request, options.forefront);- const existingRequestWithId = existingQueueById.requests.get(requestModel.id);+ const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);// We already have the request present, so we return information about it- if (existingRequestWithId) {+ if (existingRequestWithIdEntry) {+ const existingRequestWithId = await existingRequestWithIdEntry.get();existingQueueById.updateTimestamps(false);return {@@ -184,11 +189,18 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue};}- existingQueueById.requests.set(requestModel.id, requestModel);+ const newEntry = createRequestQueueStorageImplementation({+ persistStorage: existingQueueById.client.persistStorage,+ requestId: requestModel.id,+ storeDirectory: existingQueueById.client.requestQueuesDirectory,+ });++ await newEntry.update(requestModel);++ existingQueueById.requests.set(requestModel.id, newEntry);// We add 1 to pending requests if the request was not handled yetexistingQueueById.pendingRequestCount += requestModel.orderNo !== null ? 1 : 0;existingQueueById.updateTimestamps(true);- existingQueueById.updateItem(requestModel);return {requestId: requestModel.id,@@ -217,9 +229,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuefor (const model of requests) {const requestModel = this._createInternalRequest(model, options.forefront);- const existingRequestWithId = existingQueueById.requests.get(requestModel.id);+ const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);++ if (existingRequestWithIdEntry) {+ const existingRequestWithId = await existingRequestWithIdEntry.get();- if (existingRequestWithId) {result.processedRequests.push({requestId: existingRequestWithId.id,uniqueKey: existingRequestWithId.uniqueKey,@@ -230,7 +244,15 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuecontinue;}- existingQueueById.requests.set(requestModel.id, requestModel);+ const newEntry = createRequestQueueStorageImplementation({+ persistStorage: existingQueueById.client.persistStorage,+ requestId: requestModel.id,+ storeDirectory: existingQueueById.client.requestQueuesDirectory,+ });++ await newEntry.update(requestModel);++ existingQueueById.requests.set(requestModel.id, newEntry);// We add 1 to pending requests if the request was not handled yetexistingQueueById.pendingRequestCount += requestModel.orderNo !== null ? 1 : 0;result.processedRequests.push({@@ -241,7 +263,6 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuewasAlreadyHandled: false,wasAlreadyPresent: false,});- existingQueueById.updateItem(requestModel);}existingQueueById.updateTimestamps(true);@@ -259,7 +280,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueexistingQueueById.updateTimestamps(false);- const json = existingQueueById.requests.get(id)?.json;+ const json = (await existingQueueById.requests.get(id)?.get())?.json;return this._jsonToRequest(json);}@@ -278,17 +299,27 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue// First we need to check the existing request to be// able to return information about its handled state.- const existingRequest = existingQueueById.requests.get(requestModel.id);+ const existingRequestEntry = existingQueueById.requests.get(requestModel.id);// Undefined means that the request is not present in the queue.// We need to insert it, to behave the same as API.- if (!existingRequest) {+ if (!existingRequestEntry) {return this.addRequest(request, options);}+ const existingRequest = await existingRequestEntry.get();++ const newEntry = createRequestQueueStorageImplementation({+ persistStorage: existingQueueById.client.persistStorage,+ requestId: requestModel.id,+ storeDirectory: existingQueueById.client.requestQueuesDirectory,+ });++ await newEntry.update(requestModel);+// When updating the request, we need to make sure that// the handled counts are updated correctly in all cases.- existingQueueById.requests.set(requestModel.id, requestModel);+ existingQueueById.requests.set(requestModel.id, newEntry);const isRequestHandledStateChanging = typeof existingRequest.orderNo !== typeof requestModel.orderNo;const requestWasHandledBeforeUpdate = existingRequest.orderNo === null;@@ -297,7 +328,6 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueexistingQueueById.pendingRequestCount += requestWasHandledBeforeUpdate ? 1 : -1;}existingQueueById.updateTimestamps(true);- existingQueueById.updateItem(requestModel);return {requestId: requestModel.id,@@ -313,21 +343,15 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuethis.throwOnNonExisting(StorageTypes.RequestQueue);}- const request = existingQueueById.requests.get(id);+ const entry = existingQueueById.requests.get(id);++ if (entry) {+ const request = await entry.get();- if (request) {existingQueueById.requests.delete(id);existingQueueById.pendingRequestCount -= request.orderNo ? 1 : 0;existingQueueById.updateTimestamps(true);- sendWorkerMessage({- action: 'delete-entry',- data: { id },- entityType: 'requestQueues',- entityDirectory: existingQueueById.requestQueueDirectory,- id: existingQueueById.name ?? existingQueueById.id,- writeMetadata: existingQueueById.client.writeMetadata,- persistStorage: existingQueueById.client.persistStorage,- });+ await entry.delete();}}@@ -366,18 +390,6 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue});}- private updateItem(item: InternalRequest) {- sendWorkerMessage({- action: 'update-entries',- data: item,- entityType: 'requestQueues',- entityDirectory: this.requestQueueDirectory,- id: this.name ?? this.id,- writeMetadata: this.client.writeMetadata,- persistStorage: this.client.persistStorage,- });- }-private _jsonToRequest<T>(requestJson?: string): T | undefined {if (!requestJson) return undefined;const request = JSON.parse(requestJson);diff --git a/packages/memory-storage/src/utils.ts b/packages/memory-storage/src/utils.tsindex 18067abaf..455de00c1 100644--- a/packages/memory-storage/src/utils.ts+++ b/packages/memory-storage/src/utils.ts@@ -3,8 +3,6 @@ import type * as storage from '@crawlee/types';import { s } from '@sapphire/shapeshift';import { createHash } from 'node:crypto';import { REQUEST_ID_LENGTH } from './consts';-import type { InternalKeyRecord } from './resource-clients/key-value-store';-import type { InternalRequest } from './resource-clients/request-queue';/*** Removes all properties with a null value@@ -67,20 +65,13 @@ export interface WorkerData {requestQueuesDirectory: string;}-export type WorkerReceivedMessage = WorkerUpdateMetadataMessage | WorkerUpdateEntriesMessage | WorkerDeleteEntryMessage;+export type WorkerReceivedMessage = WorkerUpdateMetadataMessage;export type WorkerUpdateMetadataMessage =| MetadataUpdate<'datasets', storage.DatasetInfo>| MetadataUpdate<'keyValueStores', storage.KeyValueStoreInfo>| MetadataUpdate<'requestQueues', storage.RequestQueueInfo>;-export type WorkerUpdateEntriesMessage =- | EntriesUpdate<'datasets', [string, storage.Dictionary][]>- | EntriesUpdate<'keyValueStores', KeyValueStoreItemData>- | EntriesUpdate<'requestQueues', InternalRequest>;--export type WorkerDeleteEntryMessage = EntryDelete<'requestQueues'>;-type EntityType = 'datasets' | 'keyValueStores' | 'requestQueues';interface MetadataUpdate<Type extends EntityType, DataType> {@@ -92,30 +83,3 @@ interface MetadataUpdate<Type extends EntityType, DataType> {writeMetadata: boolean;persistStorage: boolean;}--interface EntriesUpdate<Type extends EntityType, DataType> {- entityType: Type;- id: string;- action: 'update-entries';- entityDirectory: string;- data: DataType;- writeMetadata: boolean;- persistStorage: boolean;-}--interface EntryDelete<Type extends EntityType> {- entityType: Type;- id: string;- action: 'delete-entry';- entityDirectory: string;- writeMetadata: boolean;- persistStorage: boolean;- data: {- id: string;- };-}--interface KeyValueStoreItemData {- action: 'set' | 'delete';- record: InternalKeyRecord;-}diff --git a/packages/memory-storage/src/workers/worker-utils.ts b/packages/memory-storage/src/workers/worker-utils.tsindex dbdfa6464..5953a01ca 100644--- a/packages/memory-storage/src/workers/worker-utils.ts+++ b/packages/memory-storage/src/workers/worker-utils.ts@@ -1,11 +1,11 @@import log from '@apify/log';import { ensureDir } from 'fs-extra';-import { rm, writeFile } from 'node:fs/promises';+import { writeFile } from 'node:fs/promises';import { setTimeout } from 'node:timers/promises';import { resolve } from 'node:path';import { parentPort } from 'node:worker_threads';import { lock } from 'proper-lockfile';-import type { WorkerDeleteEntryMessage, WorkerReceivedMessage, WorkerUpdateEntriesMessage, WorkerUpdateMetadataMessage } from '../utils';+import type { WorkerReceivedMessage, WorkerUpdateMetadataMessage } from '../utils';const workerLog = log.child({ prefix: 'MemoryStorageWorker' });@@ -14,12 +14,6 @@ export async function handleMessage(message: WorkerReceivedMessage & { messageIdcase 'update-metadata':await updateMetadata(message);break;- case 'update-entries':- await updateItems(message);- break;- case 'delete-entry':- await deleteEntry(message);- break;default:// We're keeping this to make eslint happy + in the event we add a new action without adding checks for it// we should be aware of them@@ -47,7 +41,7 @@ async function updateMetadata(message: WorkerUpdateMetadataMessage) {await writeFile(filePath, JSON.stringify(message.data, null, '\t'));}-async function lockAndWrite(filePath: string, data: unknown, stringify = true, retry = 10, timeout = 10): Promise<void> {+export async function lockAndWrite(filePath: string, data: unknown, stringify = true, retry = 10, timeout = 10): Promise<void> {try {const release = await lock(filePath, { realpath: false });await writeFile(filePath, stringify ? JSON.stringify(data, null, '\t') : data as Buffer);@@ -59,87 +53,3 @@ async function lockAndWrite(filePath: string, data: unknown, stringify = true, r}}}--async function updateItems(message: WorkerUpdateEntriesMessage) {- // Skip writing files to the disk if the client has the option set to false- if (!message.persistStorage) {- return;- }-- // Ensure the directory for the entity exists- const dir = message.entityDirectory;- await ensureDir(dir);-- switch (message.entityType) {- case 'requestQueues': {- // Write the entry to the file- const filePath = resolve(dir, `${message.data.id}.json`);- await lockAndWrite(filePath, message.data);- break;- }- case 'datasets': {- // Save all the new items to the disk- for (const [idx, data] of message.data) {- const filePath = resolve(dir, `${idx}.json`);- await lockAndWrite(filePath, data);- }-- break;- }- case 'keyValueStores': {- // Create files for the record- const { action, record } = message.data;-- const itemPath = resolve(dir, `${record.key}.${record.extension}`);- const itemMetadataPath = resolve(dir, `${record.key}.__metadata__.json`);-- switch (action) {- case 'delete':- await rm(itemPath, { force: true });- await rm(itemMetadataPath, { force: true });- break;- case 'set': {- await rm(itemPath, { force: true });- await rm(itemMetadataPath, { force: true });-- if (message.writeMetadata) {- await lockAndWrite(- itemMetadataPath,- {- key: record.key,- contentType: record.contentType ?? 'unknown/no content type',- extension: record.extension,- },- );- }-- await lockAndWrite(itemPath, record.value, false);-- break;- }- default:- }-- break;- }- default:- }-}--async function deleteEntry(message: WorkerDeleteEntryMessage) {- // Ensure the directory for the entity exists- const dir = message.entityDirectory;- await ensureDir(dir);-- switch (message.entityType) {- case 'requestQueues': {- // Write the entry to the file- const filePath = resolve(dir, `${message.data.id}.json`);-- await rm(filePath, { force: true });-- break;- }- default:- }-}diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.tsindex 97a9813bf..f95bb4453 100644--- a/packages/types/src/storages.ts+++ b/packages/types/src/storages.ts@@ -34,7 +34,7 @@ export interface DatasetCollectionData {*/export interface DatasetCollectionClient {list(): Promise<PaginatedList<Dataset>>;- getOrCreate(name: string): Promise<DatasetCollectionData>;+ getOrCreate(name?: string): Promise<DatasetCollectionData>;}export interface Dataset extends DatasetCollectionData {