Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(MemoryStorage): read from fs if persistStorage is enabled, ram only otherwise (#1761)

Vlad Frangue9039809a0c0
diff --git a/packages/memory-storage/src/cache-helpers.ts b/packages/memory-storage/src/cache-helpers.ts
index 2860b7e6d..ac9aed857 100644
--- a/packages/memory-storage/src/cache-helpers.ts
+++ b/packages/memory-storage/src/cache-helpers.ts
@@ -2,9 +2,12 @@ import type * as storage from '@crawlee/types';
import { access, opendir, readFile } from 'node:fs/promises';
import { extname, resolve } from 'node:path';
import mimeTypes from 'mime-types';
+import json5 from 'json5';
import type { InternalKeyRecord } from './resource-clients/key-value-store';
-import type { InternalRequest } from './resource-clients/request-queue';
import type { MemoryStorage } from './memory-storage';
+import { DatasetFileSystemEntry } from './fs/dataset/fs';
+import { KeyValueFileSystemEntry } from './fs/key-value-store/fs';
+import { RequestQueueFileSystemEntry } from './fs/request-queue/fs';
import { memoryStorageLog } from './utils';
const uuidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
@@ -32,7 +35,9 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entr
let id: string | undefined;
let name: string | undefined;
let itemCount = 0;
- const entries = new Map<string, storage.Dictionary>();
+
+ const entries = new Set<string>();
+
let createdAt = new Date();
let accessedAt = new Date();
let modifiedAt = new Date();
@@ -59,13 +64,8 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entr
continue;
}
- const fileContent = await readFile(resolve(datasetDir, entry.name), 'utf8');
- if (!fileContent) continue;
-
- const entryContent = JSON.parse(fileContent) as storage.Dictionary;
const entryName = entry.name.split('.')[0];
-
- entries.set(entryName, entryContent);
+ entries.add(entryName);
if (!hasSeenMetadataFile) {
itemCount++;
@@ -96,9 +96,12 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entr
newClient.modifiedAt = modifiedAt;
newClient.itemCount = itemCount;
- for (const [entryId, content] of entries) {
+ for (const entryId of entries.values()) {
+ // We create a file system entry instead of possibly making an in-memory one to allow the pre-included data to be used on demand
+ const entry = new DatasetFileSystemEntry({ storeDirectory: datasetDir, entityId: entryId, persistStorage: true });
+
// eslint-disable-next-line dot-notation
- newClient['datasetEntries'].set(entryId, { ...content });
+ newClient['datasetEntries'].set(entryId, entry);
}
client.datasetClientsHandled.push(newClient);
@@ -131,7 +134,10 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage
let createdAt = new Date();
let accessedAt = new Date();
let modifiedAt = new Date();
- const internalRecords = new Map<string, InternalKeyRecord>();
+
+ type FsRecord = Omit<InternalKeyRecord, 'value'>;
+ const internalRecords = new Map<string, FsRecord>();
+ let hasSeenMetadataForEntry = false;
for await (const entry of directoryEntries) {
if (entry.isFile()) {
@@ -151,49 +157,42 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage
}
if (entry.name.includes('.__metadata__.')) {
+ hasSeenMetadataForEntry = true;
+
// This is an entry's metadata file, we can use it to create/extend the record
const fileContent = await readFile(resolve(keyValueStoreDir, entry.name), 'utf8');
if (!fileContent) continue;
- const metadata = JSON.parse(fileContent) as Omit<InternalKeyRecord, 'value'>;
+ const metadata = JSON.parse(fileContent) as FsRecord;
const newRecord = {
...internalRecords.get(metadata.key),
...metadata,
- } as InternalKeyRecord;
+ } as FsRecord;
internalRecords.set(metadata.key, newRecord);
continue;
}
+ // This is an entry in the store, we can use it to create/extend the record
const fileContent = await readFile(resolve(keyValueStoreDir, entry.name));
const fileExtension = extname(entry.name);
const contentType = mimeTypes.contentType(entry.name) || 'text/plain';
const extension = mimeTypes.extension(contentType) as string;
- let finalFileContent: Buffer | string = fileContent;
-
- if (!fileExtension) {
- memoryStorageLog.warning([
- `Key-value entry "${entry.name}" for store ${entryNameOrId} does not have a file extension, assuming it as text.`,
- 'If you want to have correct interpretation of the file, you should add a file extension to the entry.',
- ].join('\n'));
- finalFileContent = fileContent.toString('utf8');
- } else if (contentType.includes('application/json')) {
+ // This is kept for backwards compatibility / to ignore invalid JSON files
+ if (contentType.includes('application/json')) {
const stringifiedJson = fileContent.toString('utf8');
+
try {
- // Try parsing the JSON ahead of time (not ideal but solves invalid files being loaded into stores)
- JSON.parse(stringifiedJson);
- finalFileContent = stringifiedJson;
+ json5.parse(stringifiedJson);
} catch {
memoryStorageLog.warning(
`Key-value entry "${entry.name}" for store ${entryNameOrId} has invalid JSON content and will be ignored from the store.`,
);
continue;
}
- } else if (contentType.includes('text/plain')) {
- finalFileContent = fileContent.toString('utf8');
}
const nameSplit = entry.name.split('.');
@@ -204,13 +203,12 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage
const key = nameSplit.join('.');
- const newRecord: InternalKeyRecord = {
+ const newRecord = {
key,
extension,
- value: finalFileContent,
contentType,
...internalRecords.get(key),
- };
+ } satisfies FsRecord;
internalRecords.set(key, newRecord);
}
@@ -239,8 +237,18 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage
newClient.modifiedAt = modifiedAt;
for (const [key, record] of internalRecords) {
+ // We create a file system entry instead of possibly making an in-memory one to allow the pre-included data to be used on demand
+ const entry = new KeyValueFileSystemEntry({ persistStorage: true, storeDirectory: keyValueStoreDir, writeMetadata: hasSeenMetadataForEntry });
+
+ // eslint-disable-next-line dot-notation
+ entry['rawRecord'] = { ...record };
+ // eslint-disable-next-line dot-notation
+ entry['filePath'] = resolve(keyValueStoreDir, `${record.key}.${record.extension}`);
+ // eslint-disable-next-line dot-notation
+ entry['fileMetadataPath'] = resolve(keyValueStoreDir, `${record.key}.__metadata__.json`);
+
// eslint-disable-next-line dot-notation
- newClient['keyValueEntries'].set(key, { ...record });
+ newClient['keyValueEntries'].set(key, entry);
}
client.keyValueStoresHandled.push(newClient);
@@ -275,7 +283,7 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryN
let modifiedAt = new Date();
let pendingRequestCount = 0;
let handledRequestCount = 0;
- const entries: InternalRequest[] = [];
+ const entries = new Set<string>();
for await (const entry of directoryEntries) {
if (entry.isFile()) {
@@ -298,11 +306,8 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryN
break;
}
default: {
- const fileContent = await readFile(resolve(requestQueueDir, entry.name), 'utf8');
- if (!fileContent) continue;
-
- const request = JSON.parse(fileContent) as InternalRequest;
- entries.push(request);
+ const entryName = entry.name.split('.')[0];
+ entries.add(entryName);
}
}
}
@@ -332,9 +337,15 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryN
newClient.pendingRequestCount = pendingRequestCount;
newClient.handledRequestCount = handledRequestCount;
- for (const entry of entries) {
+ for (const requestId of entries) {
+ const entry = new RequestQueueFileSystemEntry({
+ persistStorage: true,
+ requestId,
+ storeDirectory: requestQueueDir,
+ });
+
// eslint-disable-next-line dot-notation
- newClient['requests'].set(entry.id, entry);
+ newClient['requests'].set(requestId, entry);
}
client.requestQueuesHandled.push(newClient);
diff --git a/packages/memory-storage/src/fs/common.ts b/packages/memory-storage/src/fs/common.ts
new file mode 100644
index 000000000..13894caf0
--- /dev/null
+++ b/packages/memory-storage/src/fs/common.ts
@@ -0,0 +1,5 @@
+export interface StorageImplementation<T> {
+ get(): Promise<T>;
+ update(data: T): void | Promise<void>;
+ delete(): void | Promise<void>;
+}
diff --git a/packages/memory-storage/src/fs/dataset/fs.ts b/packages/memory-storage/src/fs/dataset/fs.ts
new file mode 100644
index 000000000..a280073d2
--- /dev/null
+++ b/packages/memory-storage/src/fs/dataset/fs.ts
@@ -0,0 +1,27 @@
+import { readFile, rm } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { ensureDir } from 'fs-extra';
+import type { CreateStorageImplementationOptions } from './index';
+import type { StorageImplementation } from '../common';
+import { lockAndWrite } from '../../workers/worker-utils';
+
+export class DatasetFileSystemEntry<Data> implements StorageImplementation<Data> {
+ private filePath: string;
+
+ constructor(options: CreateStorageImplementationOptions) {
+ this.filePath = resolve(options.storeDirectory, `${options.entityId}.json`);
+ }
+
+ async get() {
+ return JSON.parse(await readFile(this.filePath, 'utf-8'));
+ }
+
+ async update(data: Data) {
+ await ensureDir(dirname(this.filePath));
+ await lockAndWrite(this.filePath, data);
+ }
+
+ async delete() {
+ await rm(this.filePath, { force: true });
+ }
+}
diff --git a/packages/memory-storage/src/fs/dataset/index.ts b/packages/memory-storage/src/fs/dataset/index.ts
new file mode 100644
index 000000000..1f5aaa94d
--- /dev/null
+++ b/packages/memory-storage/src/fs/dataset/index.ts
@@ -0,0 +1,19 @@
+import type { Dictionary } from '@crawlee/types';
+import type { StorageImplementation } from '../common';
+import { DatasetFileSystemEntry } from './fs';
+import { DatasetMemoryEntry } from './memory';
+
+export function createDatasetStorageImplementation<Data extends Dictionary>(options: CreateStorageImplementationOptions): StorageImplementation<Data> {
+ if (options.persistStorage) {
+ return new DatasetFileSystemEntry<Data>(options);
+ }
+
+ return new DatasetMemoryEntry<Data>();
+}
+
+export interface CreateStorageImplementationOptions {
+ persistStorage: boolean;
+ storeDirectory: string;
+ /** The actual id of the file to save */
+ entityId: string;
+}
diff --git a/packages/memory-storage/src/fs/dataset/memory.ts b/packages/memory-storage/src/fs/dataset/memory.ts
new file mode 100644
index 000000000..569b77beb
--- /dev/null
+++ b/packages/memory-storage/src/fs/dataset/memory.ts
@@ -0,0 +1,17 @@
+import type { StorageImplementation } from '../common';
+
+export class DatasetMemoryEntry<Data> implements StorageImplementation<Data> {
+ private data!: Data;
+
+ async get() {
+ return this.data;
+ }
+
+ update(data: Data) {
+ this.data = data;
+ }
+
+ delete() {
+ // No-op
+ }
+}
diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.ts
new file mode 100644
index 000000000..af87497fb
--- /dev/null
+++ b/packages/memory-storage/src/fs/key-value-store/fs.ts
@@ -0,0 +1,69 @@
+import { ensureDir } from 'fs-extra';
+import { readFile, rm } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import { basename } from 'node:path/win32';
+import type { CreateStorageImplementationOptions } from '.';
+import type { InternalKeyRecord } from '../../resource-clients/key-value-store';
+import { memoryStorageLog } from '../../utils';
+import { lockAndWrite } from '../../workers/worker-utils';
+import type { StorageImplementation } from '../common';
+
+export class KeyValueFileSystemEntry implements StorageImplementation<InternalKeyRecord> {
+ private storeDirectory: string;
+ private writeMetadata: boolean;
+
+ private filePath!: string;
+ private fileMetadataPath!: string;
+ private rawRecord!: Omit<InternalKeyRecord, 'value'>;
+
+ constructor(options: CreateStorageImplementationOptions) {
+ this.storeDirectory = options.storeDirectory;
+ this.writeMetadata = options.writeMetadata;
+ }
+
+ async get(): Promise<InternalKeyRecord> {
+ let file: Buffer | string;
+
+ try {
+ file = await readFile(this.filePath);
+ } catch {
+ try {
+ // Try without extension
+ file = await readFile(resolve(this.storeDirectory, this.rawRecord.key));
+ memoryStorageLog.warning([
+ `Key-value entry "${this.rawRecord.key}" for store ${basename(this.storeDirectory)} does not have a file extension, assuming it as text.`,
+ 'If you want to have correct interpretation of the file, you should add a file extension to the entry.',
+ ].join('\n'));
+ file = file.toString('utf-8');
+ } catch {
+ // This is impossible to happen, but just in case
+ throw new Error(`Could not find file at ${this.filePath}`);
+ }
+ }
+
+ return {
+ ...this.rawRecord,
+ value: file,
+ };
+ }
+
+ async update(data: InternalKeyRecord) {
+ this.filePath ??= resolve(this.storeDirectory, `${data.key}.${data.extension}`);
+ this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);
+
+ const { value, ...rest } = data;
+ this.rawRecord = rest;
+
+ await ensureDir(dirname(this.filePath));
+ await lockAndWrite(this.filePath, value, false);
+
+ if (this.writeMetadata) {
+ await lockAndWrite(this.fileMetadataPath, JSON.stringify(rest), true);
+ }
+ }
+
+ async delete() {
+ await rm(this.filePath, { force: true });
+ await rm(this.fileMetadataPath, { force: true });
+ }
+}
diff --git a/packages/memory-storage/src/fs/key-value-store/index.ts b/packages/memory-storage/src/fs/key-value-store/index.ts
new file mode 100644
index 000000000..5ca6b0394
--- /dev/null
+++ b/packages/memory-storage/src/fs/key-value-store/index.ts
@@ -0,0 +1,18 @@
+import type { InternalKeyRecord } from '../../resource-clients/key-value-store';
+import type { StorageImplementation } from '../common';
+import { KeyValueFileSystemEntry } from './fs';
+import { KeyValueMemoryEntry } from './memory';
+
+export function createKeyValueStorageImplementation(options: CreateStorageImplementationOptions): StorageImplementation<InternalKeyRecord> {
+ if (options.persistStorage) {
+ return new KeyValueFileSystemEntry(options);
+ }
+
+ return new KeyValueMemoryEntry();
+}
+
+export interface CreateStorageImplementationOptions {
+ persistStorage: boolean;
+ storeDirectory: string;
+ writeMetadata: boolean;
+}
diff --git a/packages/memory-storage/src/fs/key-value-store/memory.ts b/packages/memory-storage/src/fs/key-value-store/memory.ts
new file mode 100644
index 000000000..bc9e9e4a0
--- /dev/null
+++ b/packages/memory-storage/src/fs/key-value-store/memory.ts
@@ -0,0 +1,18 @@
+import type { InternalKeyRecord } from '../../resource-clients/key-value-store';
+import type { StorageImplementation } from '../common';
+
+export class KeyValueMemoryEntry implements StorageImplementation<InternalKeyRecord> {
+ private data!: InternalKeyRecord;
+
+ async get() {
+ return this.data;
+ }
+
+ update(data: InternalKeyRecord) {
+ this.data = data;
+ }
+
+ delete() {
+ // No-op
+ }
+}
diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.ts
new file mode 100644
index 000000000..c3b75e300
--- /dev/null
+++ b/packages/memory-storage/src/fs/request-queue/fs.ts
@@ -0,0 +1,28 @@
+import { ensureDir } from 'fs-extra';
+import { readFile, rm } from 'node:fs/promises';
+import { dirname, resolve } from 'node:path';
+import type { CreateStorageImplementationOptions } from '.';
+import type { InternalRequest } from '../../resource-clients/request-queue';
+import { lockAndWrite } from '../../workers/worker-utils';
+import type { StorageImplementation } from '../common';
+
+export class RequestQueueFileSystemEntry implements StorageImplementation<InternalRequest> {
+ private filePath: string;
+
+ constructor(options: CreateStorageImplementationOptions) {
+ this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);
+ }
+
+ async get() {
+ return JSON.parse(await readFile(this.filePath, 'utf-8'));
+ }
+
+ async update(data: InternalRequest) {
+ await ensureDir(dirname(this.filePath));
+ await lockAndWrite(this.filePath, data);
+ }
+
+ async delete() {
+ await rm(this.filePath, { force: true });
+ }
+}
diff --git a/packages/memory-storage/src/fs/request-queue/index.ts b/packages/memory-storage/src/fs/request-queue/index.ts
new file mode 100644
index 000000000..b38bffaf9
--- /dev/null
+++ b/packages/memory-storage/src/fs/request-queue/index.ts
@@ -0,0 +1,18 @@
+import type { InternalRequest } from '../../resource-clients/request-queue';
+import type { StorageImplementation } from '../common';
+import { RequestQueueFileSystemEntry } from './fs';
+import { RequestQueueMemoryEntry } from './memory';
+
+export function createRequestQueueStorageImplementation(options: CreateStorageImplementationOptions): StorageImplementation<InternalRequest> {
+ if (options.persistStorage) {
+ return new RequestQueueFileSystemEntry(options);
+ }
+
+ return new RequestQueueMemoryEntry();
+}
+
+export interface CreateStorageImplementationOptions {
+ persistStorage: boolean;
+ storeDirectory: string;
+ requestId: string;
+}
diff --git a/packages/memory-storage/src/fs/request-queue/memory.ts b/packages/memory-storage/src/fs/request-queue/memory.ts
new file mode 100644
index 000000000..fb4e982bd
--- /dev/null
+++ b/packages/memory-storage/src/fs/request-queue/memory.ts
@@ -0,0 +1,18 @@
+import type { InternalRequest } from '../../resource-clients/request-queue';
+import type { StorageImplementation } from '../common';
+
+export class RequestQueueMemoryEntry implements StorageImplementation<InternalRequest> {
+ private data!: InternalRequest;
+
+ async get() {
+ return this.data;
+ }
+
+ update(data: InternalRequest) {
+ this.data = data;
+ }
+
+ delete() {
+ // No-op
+ }
+}
diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.ts
index d0dd3494d..00fd317db 100644
--- a/packages/memory-storage/src/resource-clients/dataset.ts
+++ b/packages/memory-storage/src/resource-clients/dataset.ts
@@ -11,6 +11,8 @@ import { StorageTypes } from '../consts';
import { BaseClient } from './common/base-client';
import { sendWorkerMessage } from '../workers/instance';
import { findOrCacheDatasetByPossibleId } from '../cache-helpers';
+import type { StorageImplementation } from '../fs/common';
+import { createDatasetStorageImplementation } from '../fs/dataset';
/**
* This is what API returns in the x-apify-pagination-limit
@@ -39,7 +41,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseCli
itemCount = 0;
datasetDirectory: string;
- private readonly datasetEntries = new Map<string, Data>();
+ private readonly datasetEntries = new Map<string, StorageImplementation<Data>>();
private readonly client: MemoryStorage;
constructor(options: DatasetClientOptions) {
@@ -141,7 +143,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseCli
for (let idx = start; idx < end; idx++) {
const entryNumber = this.generateLocalEntryName(idx);
- items.push(existingStoreById.datasetEntries.get(entryNumber)!);
+ items.push(await existingStoreById.datasetEntries.get(entryNumber)!.get());
}
existingStoreById.updateTimestamps(false);
@@ -176,23 +178,19 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseCli
for (const entry of normalized) {
const idx = this.generateLocalEntryName(++existingStoreById.itemCount);
+ const storageEntry = createDatasetStorageImplementation({
+ entityId: idx,
+ persistStorage: this.client.persistStorage,
+ storeDirectory: existingStoreById.datasetDirectory,
+ });
- existingStoreById.datasetEntries.set(idx, entry);
+ await storageEntry.update(entry);
+
+ existingStoreById.datasetEntries.set(idx, storageEntry);
addedIds.push(idx);
}
- const dataEntries: [string, Dictionary][] = addedIds.map((id) => [id, existingStoreById.datasetEntries.get(id)!]);
-
existingStoreById.updateTimestamps(true);
- sendWorkerMessage({
- data: dataEntries,
- action: 'update-entries',
- entityType: 'datasets',
- entityDirectory: existingStoreById.datasetDirectory,
- id: existingStoreById.name ?? existingStoreById.id,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
}
toDatasetInfo(): storage.DatasetInfo {
diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.ts
index 5b3ae1699..3e7eec4b2 100644
--- a/packages/memory-storage/src/resource-clients/key-value-store.ts
+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts
@@ -13,6 +13,8 @@ import { isBuffer, isStream } from '../utils';
import { BaseClient } from './common/base-client';
import { sendWorkerMessage } from '../workers/instance';
import { findOrCacheKeyValueStoreByPossibleId } from '../cache-helpers';
+import type { StorageImplementation } from '../fs/common';
+import { createKeyValueStorageImplementation } from '../fs/key-value-store';
const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
@@ -37,7 +39,7 @@ export class KeyValueStoreClient extends BaseClient {
modifiedAt = new Date();
keyValueStoreDirectory: string;
- private readonly keyValueEntries = new Map<string, InternalKeyRecord>();
+ private readonly keyValueEntries = new Map<string, StorageImplementation<InternalKeyRecord>>();
private readonly client: MemoryStorage;
constructor(options: KeyValueStoreClientOptions) {
@@ -125,7 +127,9 @@ export class KeyValueStoreClient extends BaseClient {
const items = [];
- for (const record of existingStoreById.keyValueEntries.values()) {
+ for (const storageEntry of existingStoreById.keyValueEntries.values()) {
+ const record = await storageEntry.get();
+
const size = Buffer.byteLength(record.value);
items.push({
key: record.key,
@@ -183,12 +187,14 @@ export class KeyValueStoreClient extends BaseClient {
this.throwOnNonExisting(StorageTypes.KeyValueStore);
}
- const entry = existingStoreById.keyValueEntries.get(key);
+ const storageEntry = existingStoreById.keyValueEntries.get(key);
- if (!entry) {
+ if (!storageEntry) {
return undefined;
}
+ const entry = await storageEntry.get();
+
const record: storage.KeyValueStoreRecord = {
key: entry.key,
value: entry.value,
@@ -256,28 +262,24 @@ export class KeyValueStoreClient extends BaseClient {
value = Buffer.concat(chunks);
}
- const entry = {
+ const _record = {
extension,
key,
value,
contentType,
- };
+ } satisfies InternalKeyRecord;
+
+ const entry = createKeyValueStorageImplementation({
+ persistStorage: this.client.persistStorage,
+ storeDirectory: existingStoreById.keyValueStoreDirectory,
+ writeMetadata: existingStoreById.client.writeMetadata,
+ });
+
+ await entry.update(_record);
existingStoreById.keyValueEntries.set(key, entry);
existingStoreById.updateTimestamps(true);
- sendWorkerMessage({
- action: 'update-entries',
- data: {
- action: 'set',
- record: entry,
- },
- entityType: 'keyValueStores',
- entityDirectory: existingStoreById.keyValueStoreDirectory,
- id: existingStoreById.name ?? existingStoreById.id,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
}
async deleteRecord(key: string): Promise<void> {
@@ -295,18 +297,7 @@ export class KeyValueStoreClient extends BaseClient {
if (entry) {
existingStoreById.keyValueEntries.delete(key);
existingStoreById.updateTimestamps(true);
- sendWorkerMessage({
- action: 'update-entries',
- data: {
- action: 'delete',
- record: entry,
- },
- entityType: 'keyValueStores',
- entityDirectory: existingStoreById.keyValueStoreDirectory,
- id: existingStoreById.name ?? existingStoreById.id,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
+ await entry.delete();
}
}
diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.ts
index 400039f4e..58d263528 100644
--- a/packages/memory-storage/src/resource-clients/request-queue.ts
+++ b/packages/memory-storage/src/resource-clients/request-queue.ts
@@ -10,6 +10,8 @@ import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils';
import { BaseClient } from './common/base-client';
import { sendWorkerMessage } from '../workers/instance';
import { findRequestQueueByPossibleId } from '../cache-helpers';
+import type { StorageImplementation } from '../fs/common';
+import { createRequestQueueStorageImplementation } from '../fs/request-queue';
const requestShape = s.object({
id: s.string,
@@ -54,7 +56,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
pendingRequestCount = 0;
requestQueueDirectory: string;
- private readonly requests = new Map<string, InternalRequest>();
+ private readonly requests = new Map<string, StorageImplementation<InternalRequest>>();
private readonly client: MemoryStorage;
constructor(options: RequestQueueClientOptions) {
@@ -141,11 +143,13 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
const items = [];
- for (const request of existingQueueById.requests.values()) {
+ for (const storageEntry of existingQueueById.requests.values()) {
if (items.length === limit) {
break;
}
+ const request = await storageEntry.get();
+
if (request.orderNo) {
items.push(request);
}
@@ -171,10 +175,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
const requestModel = this._createInternalRequest(request, options.forefront);
- const existingRequestWithId = existingQueueById.requests.get(requestModel.id);
+ const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);
// We already have the request present, so we return information about it
- if (existingRequestWithId) {
+ if (existingRequestWithIdEntry) {
+ const existingRequestWithId = await existingRequestWithIdEntry.get();
existingQueueById.updateTimestamps(false);
return {
@@ -184,11 +189,18 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
};
}
- existingQueueById.requests.set(requestModel.id, requestModel);
+ const newEntry = createRequestQueueStorageImplementation({
+ persistStorage: existingQueueById.client.persistStorage,
+ requestId: requestModel.id,
+ storeDirectory: existingQueueById.client.requestQueuesDirectory,
+ });
+
+ await newEntry.update(requestModel);
+
+ existingQueueById.requests.set(requestModel.id, newEntry);
// We add 1 to pending requests if the request was not handled yet
existingQueueById.pendingRequestCount += requestModel.orderNo !== null ? 1 : 0;
existingQueueById.updateTimestamps(true);
- existingQueueById.updateItem(requestModel);
return {
requestId: requestModel.id,
@@ -217,9 +229,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
for (const model of requests) {
const requestModel = this._createInternalRequest(model, options.forefront);
- const existingRequestWithId = existingQueueById.requests.get(requestModel.id);
+ const existingRequestWithIdEntry = existingQueueById.requests.get(requestModel.id);
+
+ if (existingRequestWithIdEntry) {
+ const existingRequestWithId = await existingRequestWithIdEntry.get();
- if (existingRequestWithId) {
result.processedRequests.push({
requestId: existingRequestWithId.id,
uniqueKey: existingRequestWithId.uniqueKey,
@@ -230,7 +244,15 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
continue;
}
- existingQueueById.requests.set(requestModel.id, requestModel);
+ const newEntry = createRequestQueueStorageImplementation({
+ persistStorage: existingQueueById.client.persistStorage,
+ requestId: requestModel.id,
+ storeDirectory: existingQueueById.client.requestQueuesDirectory,
+ });
+
+ await newEntry.update(requestModel);
+
+ existingQueueById.requests.set(requestModel.id, newEntry);
// We add 1 to pending requests if the request was not handled yet
existingQueueById.pendingRequestCount += requestModel.orderNo !== null ? 1 : 0;
result.processedRequests.push({
@@ -241,7 +263,6 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
wasAlreadyHandled: false,
wasAlreadyPresent: false,
});
- existingQueueById.updateItem(requestModel);
}
existingQueueById.updateTimestamps(true);
@@ -259,7 +280,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
existingQueueById.updateTimestamps(false);
- const json = existingQueueById.requests.get(id)?.json;
+ const json = (await existingQueueById.requests.get(id)?.get())?.json;
return this._jsonToRequest(json);
}
@@ -278,17 +299,27 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
// First we need to check the existing request to be
// able to return information about its handled state.
- const existingRequest = existingQueueById.requests.get(requestModel.id);
+ const existingRequestEntry = existingQueueById.requests.get(requestModel.id);
// Undefined means that the request is not present in the queue.
// We need to insert it, to behave the same as API.
- if (!existingRequest) {
+ if (!existingRequestEntry) {
return this.addRequest(request, options);
}
+ const existingRequest = await existingRequestEntry.get();
+
+ const newEntry = createRequestQueueStorageImplementation({
+ persistStorage: existingQueueById.client.persistStorage,
+ requestId: requestModel.id,
+ storeDirectory: existingQueueById.client.requestQueuesDirectory,
+ });
+
+ await newEntry.update(requestModel);
+
// When updating the request, we need to make sure that
// the handled counts are updated correctly in all cases.
- existingQueueById.requests.set(requestModel.id, requestModel);
+ existingQueueById.requests.set(requestModel.id, newEntry);
const isRequestHandledStateChanging = typeof existingRequest.orderNo !== typeof requestModel.orderNo;
const requestWasHandledBeforeUpdate = existingRequest.orderNo === null;
@@ -297,7 +328,6 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
existingQueueById.pendingRequestCount += requestWasHandledBeforeUpdate ? 1 : -1;
}
existingQueueById.updateTimestamps(true);
- existingQueueById.updateItem(requestModel);
return {
requestId: requestModel.id,
@@ -313,21 +343,15 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
this.throwOnNonExisting(StorageTypes.RequestQueue);
}
- const request = existingQueueById.requests.get(id);
+ const entry = existingQueueById.requests.get(id);
+
+ if (entry) {
+ const request = await entry.get();
- if (request) {
existingQueueById.requests.delete(id);
existingQueueById.pendingRequestCount -= request.orderNo ? 1 : 0;
existingQueueById.updateTimestamps(true);
- sendWorkerMessage({
- action: 'delete-entry',
- data: { id },
- entityType: 'requestQueues',
- entityDirectory: existingQueueById.requestQueueDirectory,
- id: existingQueueById.name ?? existingQueueById.id,
- writeMetadata: existingQueueById.client.writeMetadata,
- persistStorage: existingQueueById.client.persistStorage,
- });
+ await entry.delete();
}
}
@@ -366,18 +390,6 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
});
}
- private updateItem(item: InternalRequest) {
- sendWorkerMessage({
- action: 'update-entries',
- data: item,
- entityType: 'requestQueues',
- entityDirectory: this.requestQueueDirectory,
- id: this.name ?? this.id,
- writeMetadata: this.client.writeMetadata,
- persistStorage: this.client.persistStorage,
- });
- }
-
private _jsonToRequest<T>(requestJson?: string): T | undefined {
if (!requestJson) return undefined;
const request = JSON.parse(requestJson);
diff --git a/packages/memory-storage/src/utils.ts b/packages/memory-storage/src/utils.ts
index 18067abaf..455de00c1 100644
--- a/packages/memory-storage/src/utils.ts
+++ b/packages/memory-storage/src/utils.ts
@@ -3,8 +3,6 @@ import type * as storage from '@crawlee/types';
import { s } from '@sapphire/shapeshift';
import { createHash } from 'node:crypto';
import { REQUEST_ID_LENGTH } from './consts';
-import type { InternalKeyRecord } from './resource-clients/key-value-store';
-import type { InternalRequest } from './resource-clients/request-queue';
/**
* Removes all properties with a null value
@@ -67,20 +65,13 @@ export interface WorkerData {
requestQueuesDirectory: string;
}
-export type WorkerReceivedMessage = WorkerUpdateMetadataMessage | WorkerUpdateEntriesMessage | WorkerDeleteEntryMessage;
+export type WorkerReceivedMessage = WorkerUpdateMetadataMessage;
export type WorkerUpdateMetadataMessage =
| MetadataUpdate<'datasets', storage.DatasetInfo>
| MetadataUpdate<'keyValueStores', storage.KeyValueStoreInfo>
| MetadataUpdate<'requestQueues', storage.RequestQueueInfo>;
-export type WorkerUpdateEntriesMessage =
- | EntriesUpdate<'datasets', [string, storage.Dictionary][]>
- | EntriesUpdate<'keyValueStores', KeyValueStoreItemData>
- | EntriesUpdate<'requestQueues', InternalRequest>;
-
-export type WorkerDeleteEntryMessage = EntryDelete<'requestQueues'>;
-
type EntityType = 'datasets' | 'keyValueStores' | 'requestQueues';
interface MetadataUpdate<Type extends EntityType, DataType> {
@@ -92,30 +83,3 @@ interface MetadataUpdate<Type extends EntityType, DataType> {
writeMetadata: boolean;
persistStorage: boolean;
}
-
-interface EntriesUpdate<Type extends EntityType, DataType> {
- entityType: Type;
- id: string;
- action: 'update-entries';
- entityDirectory: string;
- data: DataType;
- writeMetadata: boolean;
- persistStorage: boolean;
-}
-
-interface EntryDelete<Type extends EntityType> {
- entityType: Type;
- id: string;
- action: 'delete-entry';
- entityDirectory: string;
- writeMetadata: boolean;
- persistStorage: boolean;
- data: {
- id: string;
- };
-}
-
-interface KeyValueStoreItemData {
- action: 'set' | 'delete';
- record: InternalKeyRecord;
-}
diff --git a/packages/memory-storage/src/workers/worker-utils.ts b/packages/memory-storage/src/workers/worker-utils.ts
index dbdfa6464..5953a01ca 100644
--- a/packages/memory-storage/src/workers/worker-utils.ts
+++ b/packages/memory-storage/src/workers/worker-utils.ts
@@ -1,11 +1,11 @@
import log from '@apify/log';
import { ensureDir } from 'fs-extra';
-import { rm, writeFile } from 'node:fs/promises';
+import { writeFile } from 'node:fs/promises';
import { setTimeout } from 'node:timers/promises';
import { resolve } from 'node:path';
import { parentPort } from 'node:worker_threads';
import { lock } from 'proper-lockfile';
-import type { WorkerDeleteEntryMessage, WorkerReceivedMessage, WorkerUpdateEntriesMessage, WorkerUpdateMetadataMessage } from '../utils';
+import type { WorkerReceivedMessage, WorkerUpdateMetadataMessage } from '../utils';
const workerLog = log.child({ prefix: 'MemoryStorageWorker' });
@@ -14,12 +14,6 @@ export async function handleMessage(message: WorkerReceivedMessage & { messageId
case 'update-metadata':
await updateMetadata(message);
break;
- case 'update-entries':
- await updateItems(message);
- break;
- case 'delete-entry':
- await deleteEntry(message);
- break;
default:
// We're keeping this to make eslint happy + in the event we add a new action without adding checks for it
// we should be aware of them
@@ -47,7 +41,7 @@ async function updateMetadata(message: WorkerUpdateMetadataMessage) {
await writeFile(filePath, JSON.stringify(message.data, null, '\t'));
}
-async function lockAndWrite(filePath: string, data: unknown, stringify = true, retry = 10, timeout = 10): Promise<void> {
+export async function lockAndWrite(filePath: string, data: unknown, stringify = true, retry = 10, timeout = 10): Promise<void> {
try {
const release = await lock(filePath, { realpath: false });
await writeFile(filePath, stringify ? JSON.stringify(data, null, '\t') : data as Buffer);
@@ -59,87 +53,3 @@ async function lockAndWrite(filePath: string, data: unknown, stringify = true, r
}
}
}
-
-async function updateItems(message: WorkerUpdateEntriesMessage) {
- // Skip writing files to the disk if the client has the option set to false
- if (!message.persistStorage) {
- return;
- }
-
- // Ensure the directory for the entity exists
- const dir = message.entityDirectory;
- await ensureDir(dir);
-
- switch (message.entityType) {
- case 'requestQueues': {
- // Write the entry to the file
- const filePath = resolve(dir, `${message.data.id}.json`);
- await lockAndWrite(filePath, message.data);
- break;
- }
- case 'datasets': {
- // Save all the new items to the disk
- for (const [idx, data] of message.data) {
- const filePath = resolve(dir, `${idx}.json`);
- await lockAndWrite(filePath, data);
- }
-
- break;
- }
- case 'keyValueStores': {
- // Create files for the record
- const { action, record } = message.data;
-
- const itemPath = resolve(dir, `${record.key}.${record.extension}`);
- const itemMetadataPath = resolve(dir, `${record.key}.__metadata__.json`);
-
- switch (action) {
- case 'delete':
- await rm(itemPath, { force: true });
- await rm(itemMetadataPath, { force: true });
- break;
- case 'set': {
- await rm(itemPath, { force: true });
- await rm(itemMetadataPath, { force: true });
-
- if (message.writeMetadata) {
- await lockAndWrite(
- itemMetadataPath,
- {
- key: record.key,
- contentType: record.contentType ?? 'unknown/no content type',
- extension: record.extension,
- },
- );
- }
-
- await lockAndWrite(itemPath, record.value, false);
-
- break;
- }
- default:
- }
-
- break;
- }
- default:
- }
-}
-
-async function deleteEntry(message: WorkerDeleteEntryMessage) {
- // Ensure the directory for the entity exists
- const dir = message.entityDirectory;
- await ensureDir(dir);
-
- switch (message.entityType) {
- case 'requestQueues': {
- // Write the entry to the file
- const filePath = resolve(dir, `${message.data.id}.json`);
-
- await rm(filePath, { force: true });
-
- break;
- }
- default:
- }
-}
diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.ts
index 97a9813bf..f95bb4453 100644
--- a/packages/types/src/storages.ts
+++ b/packages/types/src/storages.ts
@@ -34,7 +34,7 @@ export interface DatasetCollectionData {
*/
export interface DatasetCollectionClient {
list(): Promise<PaginatedList<Dataset>>;
- getOrCreate(name: string): Promise<DatasetCollectionData>;
+ getOrCreate(name?: string): Promise<DatasetCollectionData>;
}
export interface Dataset extends DatasetCollectionData {