Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactor storage .delete methods to .drop
Ondra Urban384220a10ae1
diff --git a/docs/api/KeyValueStore.md b/docs/api/KeyValueStore.mdindex 03c6945e2..fe8251d09 100644--- a/docs/api/KeyValueStore.md+++ b/docs/api/KeyValueStore.md@@ -58,7 +58,7 @@ await store.setValue('some-key', { foo: 'bar' });const value = await store.getValue('some-key');// Delete record-await store.delete('some-key');+await store.drop('some-key');```- [KeyValueStore](keyvaluestore)diff --git a/src/dataset.js b/src/dataset.jsindex e2bb2a67c..9025c8dd9 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -7,6 +7,7 @@ import LruCache from 'apify-shared/lru_cache';import { checkParamOrThrow } from 'apify-client/build/utils';import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from 'apify-shared/consts';import { apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists } from './utils';+import log from 'apify-shared/log';export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;export const LOCAL_FILENAME_DIGITS = 9;@@ -417,15 +418,17 @@ export class Dataset {** @return {Promise}*/- delete() {- return datasets- .deleteDataset({- datasetId: this.datasetId,- })- .then(() => {- datasetsCache.remove(this.datasetId);- if (this.datasetName) datasetsCache.remove(this.datasetName);- });+ async drop() {+ await datasets.deleteDataset({ datasetId: this.datasetId });+ datasetsCache.remove(this.datasetId);+ if (this.datasetName) datasetsCache.remove(this.datasetName);+ }++ /** @ignore */+ async delete() {+ log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '+ + 'This is to make it more obvious to users that the function deletes the dataset and not individual records in the dataset.');+ await this.drop();}}@@ -577,13 +580,17 @@ export class DatasetLocal {return memo;}- delete() {- return this.initializationPromise- .then(() => emptyDirPromised(this.localStoragePath))- .then(() => {- this._updateMetadata(true);- datasetsCache.remove(this.datasetId);- });+ async drop() {+ await this.initializationPromise;+ await emptyDirPromised(this.localStoragePath);+ this._updateMetadata(true);+ datasetsCache.remove(this.datasetId);+ }++ async delete() {+ log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '+ + 'This is to make it more obvious to users that the function deletes the dataset and not individual records in the dataset.');+ await this.drop();}/**diff --git a/src/key_value_store.js b/src/key_value_store.jsindex 76ae52fec..f757bc47b 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -7,6 +7,7 @@ import LruCache from 'apify-shared/lru_cache';import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, KEY_VALUE_STORE_KEYS } from 'apify-shared/consts';import { jsonStringifyExtended } from 'apify-shared/utilities';+import log from 'apify-shared/log';import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';import {addCharsetToContentType, apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists,@@ -153,8 +154,8 @@ export const maybeStringify = (value, options) => {* // text data returned as a string and other data is returned as binary buffer* const value = await store.getValue('some-key');*- * // Delete record- * await store.delete('some-key');+ * // Drop (delete) the store+ * await store.drop();* ```* @hideconstructor*/@@ -285,15 +286,17 @@ export class KeyValueStore {** @return {Promise}*/- delete() {- return keyValueStores- .deleteStore({- storeId: this.storeId,- })- .then(() => {- storesCache.remove(this.storeId);- if (this.storeName) storesCache.remove(this.storeName);- });+ async drop() {+ await keyValueStores.deleteStore({ storeId: this.storeId });+ storesCache.remove(this.storeId);+ if (this.storeName) storesCache.remove(this.storeName);+ }++ /** @ignore */+ async delete() {+ log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '+ + 'This is to make it more obvious to users that the function deletes the key-value store and not individual records in the store.');+ await this.drop();}/**@@ -421,6 +424,12 @@ export class KeyValueStoreLocal {}async delete() {+ log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '+ + 'This is to make it more obvious to users that the function deletes the key-value store and not individual records in the store.');+ await this.drop();+ }++ async drop() {await this.initializationPromise;await emptyDirPromised(this.localStoragePath);diff --git a/src/request_queue.js b/src/request_queue.jsindex aca12d93b..d4f6ea47a 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -640,7 +640,7 @@ export class RequestQueue {** @return {Promise}*/- async delete() {+ async drop() {await requestQueues.deleteQueue({queueId: this.queueId,});@@ -649,6 +649,13 @@ export class RequestQueue {if (this.queueName) queuesCache.remove(this.queueName);}+ /** @ignore */+ async delete() {+ log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '+ + 'This is to make it more obvious to users that the function deletes the request queue and not individual records in the queue.');+ await this.drop();+ }+/*** Returns the number of handled requests.*@@ -994,11 +1001,17 @@ export class RequestQueueLocal {return this.pendingCount === 0;}- async delete() {+ async drop() {await emptyDirPromised(this.localStoragePath);queuesCache.remove(this.queueId);}+ async delete() {+ log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '+ + 'This is to make it more obvious to users that the function deletes the request queue and not individual records in the queue.');+ await this.drop();+ }+async handledCount() {const { handledRequestCount } = await this.getInfo();return handledRequestCount;diff --git a/test/dataset.js b/test/dataset.jsindex 3f84ea83e..fb5e28d54 100644--- a/test/dataset.js+++ b/test/dataset.js@@ -50,7 +50,7 @@ describe('dataset', () => {// Delete works.const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-dataset');expectDirNonEmpty(datasetDir);- await newDataset.delete();+ await newDataset.drop();expectDirEmpty(datasetDir);});@@ -344,7 +344,7 @@ describe('dataset', () => {.once().withArgs({ datasetId: 'some-id' }).returns(Promise.resolve());- await dataset.delete();+ await dataset.drop();mock.verify();mock.restore();@@ -375,7 +375,7 @@ describe('dataset', () => {.once().withArgs({ datasetId: 'some-id' }).returns(Promise.resolve());- await dataset.delete();+ await dataset.drop();mock.verify();mock.restore();@@ -408,7 +408,7 @@ describe('dataset', () => {.once().withArgs({ datasetId: 'some-id' }).returns(Promise.resolve());- await dataset.delete();+ await dataset.drop();mock.verify();mock.restore();@@ -429,7 +429,7 @@ describe('dataset', () => {.once().withArgs({ datasetId: 'some-id' }).returns(Promise.resolve());- await dataset.delete();+ await dataset.drop();mock.verify();mock.restore();});@@ -454,7 +454,7 @@ describe('dataset', () => {.once().withArgs({ datasetId: 'some-id' }).returns(Promise.resolve());- await dataset.delete();+ await dataset.drop();mock.verify();mock.restore();});diff --git a/test/key_value_store.js b/test/key_value_store.jsindex 4b6493748..63b9e113f 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -96,7 +96,7 @@ describe('KeyValueStore', () => {// Delete works.const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-store-id');expectDirNonEmpty(storeDir);- await store.delete();+ await store.drop();expectDirEmpty(storeDir);});});@@ -148,7 +148,7 @@ describe('KeyValueStore', () => {storeId: 'some-id-1',}).returns(Promise.resolve());- await store.delete();+ await store.drop();mock.verify();mock.restore();@@ -467,7 +467,7 @@ describe('KeyValueStore', () => {// Delete works.const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, storeId);expectDirNonEmpty(storeDir);- await store.delete();+ await store.drop();expectDirEmpty(storeDir);});});diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.jsindex 3ab190699..d1be8cbf6 100644--- a/test/puppeteer_pool.js+++ b/test/puppeteer_pool.js@@ -226,8 +226,6 @@ describe('PuppeteerPool', () => {await pool.destroy();});- // Test started failing on 6.10.2018. Probably some change upstream.- // Disabling the feature until resolved.it('supports recycleDiskCache option', async () => {const pool = new Apify.PuppeteerPool({maxOpenPagesPerInstance: 1,diff --git a/test/request_queue.js b/test/request_queue.jsindex c9c33c142..538bba7e2 100644--- a/test/request_queue.js+++ b/test/request_queue.js@@ -111,7 +111,7 @@ describe('RequestQueue', () => {// Delete it.const queueDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-queue-0');expectDirNonEmpty(queueDir);- await queue.delete();+ await queue.drop();expectDirEmpty(queueDir);});@@ -457,7 +457,7 @@ describe('RequestQueue', () => {queueId: 'some-id',}).returns(Promise.resolve());- await queue.delete();+ await queue.drop();mock.verify();mock.restore();