Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: allow disabling storage persistence (#1539)
Vlad Franguf65e3c6a7e1e
diff --git a/packages/core/src/configuration.ts b/packages/core/src/configuration.tsindex 459e3f741..dcd522c83 100644--- a/packages/core/src/configuration.ts+++ b/packages/core/src/configuration.ts@@ -148,10 +148,17 @@ export interface ConfigurationOptions {/*** Sets the log level to the given value.*- * Alternative to `CRAWLEE_LOG_LEVEL`.+ * Alternative to `CRAWLEE_LOG_LEVEL` environment variable.* @default 'INFO'*/logLevel?: LogLevel | LogLevel[keyof LogLevel];++ /**+ * Defines whether the storage client used should persist the data it stores.+ *+ * Alternative to `CRAWLEE_PERSIST_STORAGE` environment variable.+ */+ persistStorage?: boolean;}/**@@ -203,7 +210,9 @@ export interface ConfigurationOptions {* `defaultDatasetId` | `CRAWLEE_DEFAULT_DATASET_ID` | `'default'`* `defaultKeyValueStoreId` | `CRAWLEE_DEFAULT_KEY_VALUE_STORE_ID` | `'default'`* `defaultRequestQueueId` | `CRAWLEE_DEFAULT_REQUEST_QUEUE_ID` | `'default'`- * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60e3`+ * `persistStateIntervalMillis` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | `60_000`+ * `purgeOnStart` | `CRAWLEE_PURGE_ON_START` | `true`+ * `persistStorage` | `CRAWLEE_PERSIST_STORAGE` | `true`** ## Advanced Configuration Options*@@ -213,6 +222,8 @@ export interface ConfigurationOptions {* `xvfb` | `CRAWLEE_XVFB` | -* `chromeExecutablePath` | `CRAWLEE_CHROME_EXECUTABLE_PATH` | -* `defaultBrowserPath` | `CRAWLEE_DEFAULT_BROWSER_PATH` | -+ * `disableBrowserSandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | -+ * `availableMemoryRatio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25`*/export class Configuration {/**@@ -233,9 +244,10 @@ export class Configuration {CRAWLEE_DEFAULT_BROWSER_PATH: 'defaultBrowserPath',CRAWLEE_DISABLE_BROWSER_SANDBOX: 'disableBrowserSandbox',CRAWLEE_LOG_LEVEL: 'logLevel',+ CRAWLEE_PERSIST_STORAGE: 'persistStorage',};- protected static BOOLEAN_VARS = ['purgeOnStart', 'headless', 'xvfb', 'disableBrowserSandbox'];+ protected static BOOLEAN_VARS = ['purgeOnStart', 'headless', 'xvfb', 'disableBrowserSandbox', 'persistStorage'];protected static INTEGER_VARS = ['memoryMbytes', 'persistStateIntervalMillis', 'systemInfoIntervalMillis'];@@ -251,6 +263,7 @@ export class Configuration {headless: true,persistStateIntervalMillis: 60_000,systemInfoIntervalMillis: 60_000,+ persistStorage: true,};/**@@ -380,7 +393,11 @@ export class Configuration {return this.services.get(cacheKey) as MemoryStorage;}- const storage = new MemoryStorage(options);+ const storage = new MemoryStorage({+ persistStorage: this.get('persistStorage'),+ // Override persistStorage if user provides it via storageClientOptions+ ...options,+ });this.services.set(cacheKey, storage);return storage;diff --git a/packages/memory-storage/src/memory-storage.ts b/packages/memory-storage/src/memory-storage.tsindex c45db6ad9..acf3f4bcc 100644--- a/packages/memory-storage/src/memory-storage.ts+++ b/packages/memory-storage/src/memory-storage.ts@@ -26,6 +26,14 @@ export interface MemoryStorageOptions {* @default process.env.DEBUG?.includes('*') ?? process.env.DEBUG?.includes('crawlee:memory-storage') ?? false*/writeMetadata?: boolean;++ /**+ * Whether the memory storage should also write its stored content to the disk.+ *+ * You can also disable this by setting the `CRAWLEE_PERSIST_STORAGE` environment variable to `false`.+ * @default true+ */+ persistStorage?: boolean;}export class MemoryStorage implements storage.StorageClient {@@ -34,6 +42,7 @@ export class MemoryStorage implements storage.StorageClient {readonly keyValueStoresDirectory: string;readonly requestQueuesDirectory: string;readonly writeMetadata: boolean;+ readonly persistStorage: boolean;readonly keyValueStoresHandled: KeyValueStoreClient[] = [];readonly datasetClientsHandled: DatasetClient[] = [];@@ -43,6 +52,7 @@ export class MemoryStorage implements storage.StorageClient {s.object({localDataDirectory: s.string.optional,writeMetadata: s.boolean.optional,+ persistStorage: s.boolean.optional,}).parse(options);// v3.0.0 used `crawlee_storage` as the default, we changed this in v3.0.1 to just `storage`,@@ -61,6 +71,8 @@ export class MemoryStorage implements storage.StorageClient {this.keyValueStoresDirectory = resolve(this.localDataDirectory, 'key_value_stores');this.requestQueuesDirectory = resolve(this.localDataDirectory, 'request_queues');this.writeMetadata = options.writeMetadata ?? process.env.DEBUG?.includes('*') ?? process.env.DEBUG?.includes('crawlee:memory-storage') ?? false;+ this.persistStorage = options.persistStorage+ ?? (process.env.CRAWLEE_PERSIST_STORAGE ? !['false', '0', ''].includes(process.env.CRAWLEE_PERSIST_STORAGE!) : true);initWorkerIfNeeded();}diff --git a/packages/memory-storage/src/resource-clients/dataset-collection.ts b/packages/memory-storage/src/resource-clients/dataset-collection.tsindex 3de8688e5..652363d84 100644--- a/packages/memory-storage/src/resource-clients/dataset-collection.ts+++ b/packages/memory-storage/src/resource-clients/dataset-collection.ts@@ -57,6 +57,7 @@ export class DatasetCollectionClient implements storage.DatasetCollectionClientid: datasetInfo.name ?? datasetInfo.id,data: datasetInfo,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});return datasetInfo;diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.tsindex fad2ce247..a837d709e 100644--- a/packages/memory-storage/src/resource-clients/dataset.ts+++ b/packages/memory-storage/src/resource-clients/dataset.ts@@ -188,6 +188,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseClientityDirectory: existingStoreById.datasetDirectory,id: existingStoreById.name ?? existingStoreById.id,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});}@@ -260,6 +261,7 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseClientityDirectory: this.datasetDirectory,id: this.name ?? this.id,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});}}diff --git a/packages/memory-storage/src/resource-clients/key-value-store-collection.ts b/packages/memory-storage/src/resource-clients/key-value-store-collection.tsindex 0ab5f0e96..0bbc8c083 100644--- a/packages/memory-storage/src/resource-clients/key-value-store-collection.ts+++ b/packages/memory-storage/src/resource-clients/key-value-store-collection.ts@@ -57,6 +57,7 @@ export class KeyValueStoreCollectionClient implements storage.KeyValueStoreColleid: kvStoreInfo.name ?? kvStoreInfo.id,data: kvStoreInfo,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});return kvStoreInfo;diff --git a/packages/memory-storage/src/resource-clients/key-value-store.ts b/packages/memory-storage/src/resource-clients/key-value-store.tsindex bcc0b73c5..d6515d05b 100644--- a/packages/memory-storage/src/resource-clients/key-value-store.ts+++ b/packages/memory-storage/src/resource-clients/key-value-store.ts@@ -276,6 +276,7 @@ export class KeyValueStoreClient extends BaseClient {entityDirectory: existingStoreById.keyValueStoreDirectory,id: existingStoreById.name ?? existingStoreById.id,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});}@@ -304,6 +305,7 @@ export class KeyValueStoreClient extends BaseClient {entityDirectory: existingStoreById.keyValueStoreDirectory,id: existingStoreById.name ?? existingStoreById.id,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});}}@@ -334,6 +336,7 @@ export class KeyValueStoreClient extends BaseClient {entityDirectory: this.keyValueStoreDirectory,id: this.name ?? this.id,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});}}diff --git a/packages/memory-storage/src/resource-clients/request-queue-collection.ts b/packages/memory-storage/src/resource-clients/request-queue-collection.tsindex ba91f010c..da3b9543a 100644--- a/packages/memory-storage/src/resource-clients/request-queue-collection.ts+++ b/packages/memory-storage/src/resource-clients/request-queue-collection.ts@@ -57,6 +57,7 @@ export class RequestQueueCollectionClient implements storage.RequestQueueCollectid: queueInfo.name ?? queueInfo.id,data: queueInfo,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});return queueInfo;diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.tsindex 40e2ce8ac..b4dff3a6f 100644--- a/packages/memory-storage/src/resource-clients/request-queue.ts+++ b/packages/memory-storage/src/resource-clients/request-queue.ts@@ -326,6 +326,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueentityDirectory: existingQueueById.requestQueueDirectory,id: existingQueueById.name ?? existingQueueById.id,writeMetadata: existingQueueById.client.writeMetadata,+ persistStorage: existingQueueById.client.persistStorage,});}}@@ -361,6 +362,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueentityDirectory: this.requestQueueDirectory,id: this.name ?? this.id,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});}@@ -372,6 +374,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQueentityDirectory: this.requestQueueDirectory,id: this.name ?? this.id,writeMetadata: this.client.writeMetadata,+ persistStorage: this.client.persistStorage,});}diff --git a/packages/memory-storage/src/utils.ts b/packages/memory-storage/src/utils.tsindex c6875c2c6..18067abaf 100644--- a/packages/memory-storage/src/utils.ts+++ b/packages/memory-storage/src/utils.ts@@ -90,6 +90,7 @@ interface MetadataUpdate<Type extends EntityType, DataType> {entityDirectory: string;data: DataType;writeMetadata: boolean;+ persistStorage: boolean;}interface EntriesUpdate<Type extends EntityType, DataType> {@@ -99,6 +100,7 @@ interface EntriesUpdate<Type extends EntityType, DataType> {entityDirectory: string;data: DataType;writeMetadata: boolean;+ persistStorage: boolean;}interface EntryDelete<Type extends EntityType> {@@ -107,6 +109,7 @@ interface EntryDelete<Type extends EntityType> {action: 'delete-entry';entityDirectory: string;writeMetadata: boolean;+ persistStorage: boolean;data: {id: string;};diff --git a/packages/memory-storage/src/workers/worker-utils.ts b/packages/memory-storage/src/workers/worker-utils.tsindex 281e34b81..dbdfa6464 100644--- a/packages/memory-storage/src/workers/worker-utils.ts+++ b/packages/memory-storage/src/workers/worker-utils.ts@@ -33,15 +33,15 @@ export async function handleMessage(message: WorkerReceivedMessage & { messageId}async function updateMetadata(message: WorkerUpdateMetadataMessage) {- // Ensure the directory for the entity exists- const dir = message.entityDirectory;- await ensureDir(dir);-// Skip writing the actual metadata file. This is done after ensuring the directory exists so we have the directory presentif (!message.writeMetadata) {return;}+ // Ensure the directory for the entity exists+ const dir = message.entityDirectory;+ await ensureDir(dir);+// Write the metadata to the fileconst filePath = resolve(dir, '__metadata__.json');await writeFile(filePath, JSON.stringify(message.data, null, '\t'));@@ -61,6 +61,11 @@ async function lockAndWrite(filePath: string, data: unknown, stringify = true, r}async function updateItems(message: WorkerUpdateEntriesMessage) {+ // Skip writing files to the disk if the client has the option set to false+ if (!message.persistStorage) {+ return;+ }+// Ensure the directory for the entity existsconst dir = message.entityDirectory;await ensureDir(dir);diff --git a/packages/memory-storage/test/__shared__.ts b/packages/memory-storage/test/__shared__.tsnew file mode 100644index 000000000..dbfce6d2d--- /dev/null+++ b/packages/memory-storage/test/__shared__.ts@@ -0,0 +1,11 @@+import { access } from 'node:fs/promises';+import { setTimeout } from 'node:timers/promises';++export async function waitTillWrittenToDisk(path: string): Promise<void> {+ try {+ await access(path);+ } catch {+ await setTimeout(50);+ return waitTillWrittenToDisk(path);+ }+}diff --git a/packages/memory-storage/test/no-writing-to-disk.test.ts b/packages/memory-storage/test/no-writing-to-disk.test.tsnew file mode 100644index 000000000..32397d90a--- /dev/null+++ b/packages/memory-storage/test/no-writing-to-disk.test.ts@@ -0,0 +1,57 @@+import { readdir, rm } from 'node:fs/promises';+import { resolve } from 'node:path';+import { MemoryStorage } from '@crawlee/memory-storage';+import { waitTillWrittenToDisk } from './__shared__';++describe('persistStorage option', () => {+ const tmpLocation = resolve(__dirname, './tmp/no-writing-to-disk');++ afterAll(async () => {+ await rm(tmpLocation, { force: true, recursive: true });+ });++ describe('when false and writeMetadata is also false', () => {+ const localDataDirectory = resolve(tmpLocation, './no-metadata');+ const storage = new MemoryStorage({+ localDataDirectory,+ persistStorage: false,+ });++ test('creating a key-value pair in a key-value store should not write data to the disk', async () => {+ const keyValueStoreInfo = await storage.keyValueStores().getOrCreate();++ const keyValueStore = storage.keyValueStore(keyValueStoreInfo.id);+ await keyValueStore.setRecord({ key: 'foo', value: 'test' });++ // We check that reading the directory for the store throws an error, which means it wasn't created on disk+ await expect(() => readdir(localDataDirectory)).rejects.toThrow();+ });+ });++ describe('when false and writeMetadata is true', () => {+ const localDataDirectory = resolve(tmpLocation, './with-metadata');+ const storage = new MemoryStorage({+ localDataDirectory,+ persistStorage: false,+ writeMetadata: true,+ });++ test('creating a key-value pair in a key-value store should not write data to the disk, but it should write the __metadata__ file', async () => {+ const keyValueStoreInfo = await storage.keyValueStores().getOrCreate();++ const keyValueStore = storage.keyValueStore(keyValueStoreInfo.id);+ await keyValueStore.setRecord({ key: 'foo', value: 'test' });++ const storePath = resolve(storage.keyValueStoresDirectory, `${keyValueStoreInfo.id}`);++ await waitTillWrittenToDisk(storePath);++ const directoryFiles = await readdir(storePath);++ expect(directoryFiles).toHaveLength(1);+ expect(directoryFiles).toEqual([+ '__metadata__.json',+ ]);+ });+ });+});diff --git a/packages/memory-storage/test/write-metadata.test.ts b/packages/memory-storage/test/write-metadata.test.tsindex c31e441bc..56a2eff78 100644--- a/packages/memory-storage/test/write-metadata.test.ts+++ b/packages/memory-storage/test/write-metadata.test.ts@@ -1,16 +1,7 @@import { MemoryStorage } from '@crawlee/memory-storage';-import { access, readdir, rm } from 'node:fs/promises';+import { readdir, rm } from 'node:fs/promises';import { resolve } from 'node:path';-import { setTimeout } from 'node:timers/promises';--async function waitTillWrittenToDisk(path: string): Promise<void> {- try {- await access(path);- } catch {- await setTimeout(50);- return waitTillWrittenToDisk(path);- }-}+import { waitTillWrittenToDisk } from './__shared__';describe('writeMetadata option', () => {const tmpLocation = resolve(__dirname, './tmp/write-metadata-tests');@@ -29,11 +20,9 @@ describe('writeMetadata option', () => {test('creating a data store should not write __metadata__.json file', async () => {const keyValueStore = await storage.keyValueStores().getOrCreate();const expectedPath = resolve(storage.keyValueStoresDirectory, `${keyValueStore.id}`);- await waitTillWrittenToDisk(expectedPath);-- const directoryFiles = await readdir(expectedPath);- expect(directoryFiles).toHaveLength(0);+ // We check that reading the directory for the store throws an error, which means it wasn't created on disk+ await expect(() => readdir(expectedPath)).rejects.toThrow();});test('creating a key-value pair in a key-value store should not write __metadata__.json file for the value', async () => {