Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

refactor: make `purgeDefaultStorages` accept `options` with `onlyPurgeOnce` (#2044)

Sviatozar Petrenko8d6946c8ba41
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 90ff27c73..afc1f7ef7 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -775,7 +775,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.stats.reset();
await this.stats.resetStore();
- await purgeDefaultStorages();
+ await purgeDefaultStorages({ onlyPurgeOnce: true });
if (requests) {
await this.addRequests(requests, options);
diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.ts
index 6388f78e4..8bfc94c41 100644
--- a/packages/core/src/storages/dataset.ts
+++ b/packages/core/src/storages/dataset.ts
@@ -531,7 +531,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
options.config ??= Configuration.getGlobalConfig();
options.storageClient ??= options.config.getStorageClient();
- await purgeDefaultStorages(options.config, options.storageClient);
+ await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
const manager = StorageManager.getManager<Dataset<Data>>(this, options.config);
diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.ts
index 47f28b3e8..6d013a856 100644
--- a/packages/core/src/storages/key_value_store.ts
+++ b/packages/core/src/storages/key_value_store.ts
@@ -417,7 +417,7 @@ export class KeyValueStore {
options.config ??= Configuration.getGlobalConfig();
options.storageClient ??= options.config.getStorageClient();
- await purgeDefaultStorages(options.config, options.storageClient);
+ await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
const manager = StorageManager.getManager(this, options.config);
diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.ts
index 443bc777c..fc4cf5b08 100644
--- a/packages/core/src/storages/request_list.ts
+++ b/packages/core/src/storages/request_list.ts
@@ -345,7 +345,7 @@ export class RequestList {
}
this.isLoading = true;
- await purgeDefaultStorages();
+ await purgeDefaultStorages({ onlyPurgeOnce: true });
const [state, persistedRequests] = await this._loadStateAndPersistedRequests();
diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.ts
index 89c1c39b0..36a2ee4b7 100644
--- a/packages/core/src/storages/request_queue.ts
+++ b/packages/core/src/storages/request_queue.ts
@@ -954,7 +954,7 @@ export class RequestQueue {
options.config ??= Configuration.getGlobalConfig();
options.storageClient ??= options.config.getStorageClient();
- await purgeDefaultStorages(options.config, options.storageClient);
+ await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
const manager = StorageManager.getManager(this, options.config);
const queue = await manager.openStorage(queueIdOrName, options.storageClient);
diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.ts
index 651531da8..7140b54c5 100644
--- a/packages/core/src/storages/utils.ts
+++ b/packages/core/src/storages/utils.ts
@@ -3,6 +3,12 @@ import type { Dictionary, StorageClient } from '@crawlee/types';
import { KeyValueStore } from './key_value_store';
import { Configuration } from '../configuration';
+interface PurgeDefaultStorageOptions {
+ onlyPurgeOnce?: boolean;
+ config?: Configuration;
+ client?: StorageClient;
+}
+
/**
* Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
* Purging will remove all the files in all storages except for INPUT.json in the default KV store.
@@ -16,10 +22,39 @@ import { Configuration } from '../configuration';
* this method will make sure the storage is purged only once for a given execution context, so it is safe to call
* it multiple times.
*/
-export async function purgeDefaultStorages(config = Configuration.getGlobalConfig(), client: StorageClient = config.getStorageClient()) {
+export async function purgeDefaultStorages(config?: Configuration, client?: StorageClient): Promise<void>;
+/**
+ * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.
+ * Purging will remove all the files in all storages except for INPUT.json in the default KV store.
+ *
+ * Purging of storages is happening automatically when we run our crawler (or when we open some storage
+ * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}
+ * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.
+ *
+ * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words
+ * it will call the `purge` method of the underlying storage implementation we are currently using. In addition,
+ * this method will make sure the storage is purged only once for a given execution context, so it is safe to call
+ * it multiple times, unless you set `onlyPurgeOnce` to `false` in the `options` object
+ */
+export async function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;
+export async function purgeDefaultStorages(
+ configOrOptions?: Configuration | PurgeDefaultStorageOptions,
+ client?: StorageClient,
+) {
+ const options: PurgeDefaultStorageOptions = configOrOptions instanceof Configuration ? {
+ client,
+ config: configOrOptions,
+ } : configOrOptions ?? {};
+ const {
+ config = Configuration.getGlobalConfig(),
+ onlyPurgeOnce = true,
+ } = options;
+ ({ client = config.getStorageClient() } = options);
+
const casted = client as StorageClient & { __purged?: boolean };
- if (config.get('purgeOnStart') && !casted.__purged) {
+ // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start
+ if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {
casted.__purged = true;
await casted.purge?.();
}