Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: make `purgeDefaultStorages` accept `options` with `onlyPurgeOnce` (#2044)
Sviatozar Petrenko8d6946c8ba41
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 90ff27c73..afc1f7ef7 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -775,7 +775,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.stats.reset();await this.stats.resetStore();- await purgeDefaultStorages();+ await purgeDefaultStorages({ onlyPurgeOnce: true });if (requests) {await this.addRequests(requests, options);diff --git a/packages/core/src/storages/dataset.ts b/packages/core/src/storages/dataset.tsindex 6388f78e4..8bfc94c41 100644--- a/packages/core/src/storages/dataset.ts+++ b/packages/core/src/storages/dataset.ts@@ -531,7 +531,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {options.config ??= Configuration.getGlobalConfig();options.storageClient ??= options.config.getStorageClient();- await purgeDefaultStorages(options.config, options.storageClient);+ await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });const manager = StorageManager.getManager<Dataset<Data>>(this, options.config);diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 47f28b3e8..6d013a856 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -417,7 +417,7 @@ export class KeyValueStore {options.config ??= Configuration.getGlobalConfig();options.storageClient ??= options.config.getStorageClient();- await purgeDefaultStorages(options.config, options.storageClient);+ await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });const manager = StorageManager.getManager(this, options.config);diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 443bc777c..fc4cf5b08 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -345,7 +345,7 @@ export class RequestList {}this.isLoading = true;- await purgeDefaultStorages();+ await purgeDefaultStorages({ onlyPurgeOnce: true });const [state, persistedRequests] = await this._loadStateAndPersistedRequests();diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 89c1c39b0..36a2ee4b7 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -954,7 +954,7 @@ export class RequestQueue {options.config ??= Configuration.getGlobalConfig();options.storageClient ??= options.config.getStorageClient();- await purgeDefaultStorages(options.config, options.storageClient);+ await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });const manager = StorageManager.getManager(this, options.config);const queue = await manager.openStorage(queueIdOrName, options.storageClient);diff --git a/packages/core/src/storages/utils.ts b/packages/core/src/storages/utils.tsindex 651531da8..7140b54c5 100644--- a/packages/core/src/storages/utils.ts+++ b/packages/core/src/storages/utils.ts@@ -3,6 +3,12 @@ import type { Dictionary, StorageClient } from '@crawlee/types';import { KeyValueStore } from './key_value_store';import { Configuration } from '../configuration';+interface PurgeDefaultStorageOptions {+ onlyPurgeOnce?: boolean;+ config?: Configuration;+ client?: StorageClient;+}+/*** Cleans up the local storage folder (defaults to `./storage`) created when running code locally.* Purging will remove all the files in all storages except for INPUT.json in the default KV store.@@ -16,10 +22,39 @@ import { Configuration } from '../configuration';* this method will make sure the storage is purged only once for a given execution context, so it is safe to call* it multiple times.*/-export async function purgeDefaultStorages(config = Configuration.getGlobalConfig(), client: StorageClient = config.getStorageClient()) {+export async function purgeDefaultStorages(config?: Configuration, client?: StorageClient): Promise<void>;+/**+ * Cleans up the local storage folder (defaults to `./storage`) created when running code locally.+ * Purging will remove all the files in all storages except for INPUT.json in the default KV store.+ *+ * Purging of storages is happening automatically when we run our crawler (or when we open some storage+ * explicitly, e.g. via `RequestList.open()`). We can disable that via `purgeOnStart` {@apilink Configuration}+ * option or by setting `CRAWLEE_PURGE_ON_START` environment variable to `0` or `false`.+ *+ * This is a shortcut for running (optional) `purge` method on the StorageClient interface, in other words+ * it will call the `purge` method of the underlying storage implementation we are currently using. In addition,+ * this method will make sure the storage is purged only once for a given execution context, so it is safe to call+ * it multiple times, unless you set `onlyPurgeOnce` to `false` in the `options` object+ */+export async function purgeDefaultStorages(options?: PurgeDefaultStorageOptions): Promise<void>;+export async function purgeDefaultStorages(+ configOrOptions?: Configuration | PurgeDefaultStorageOptions,+ client?: StorageClient,+) {+ const options: PurgeDefaultStorageOptions = configOrOptions instanceof Configuration ? {+ client,+ config: configOrOptions,+ } : configOrOptions ?? {};+ const {+ config = Configuration.getGlobalConfig(),+ onlyPurgeOnce = true,+ } = options;+ ({ client = config.getStorageClient() } = options);+const casted = client as StorageClient & { __purged?: boolean };- if (config.get('purgeOnStart') && !casted.__purged) {+ // if `onlyPurgeOnce` is true, will purge anytime this function is called, otherwise - only on start+ if (!onlyPurgeOnce || (config.get('purgeOnStart') && !casted.__purged)) {casted.__purged = true;await casted.purge?.();}