Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Refactor storage .delete methods to .drop

Ondra Urban384220a10ae1
diff --git a/docs/api/KeyValueStore.md b/docs/api/KeyValueStore.md
index 03c6945e2..fe8251d09 100644
--- a/docs/api/KeyValueStore.md
+++ b/docs/api/KeyValueStore.md
@@ -58,7 +58,7 @@ await store.setValue('some-key', { foo: 'bar' });
const value = await store.getValue('some-key');
// Delete record
-await store.delete('some-key');
+await store.drop('some-key');
```
- [KeyValueStore](keyvaluestore)
diff --git a/src/dataset.js b/src/dataset.js
index e2bb2a67c..9025c8dd9 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -7,6 +7,7 @@ import LruCache from 'apify-shared/lru_cache';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from 'apify-shared/consts';
import { apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists } from './utils';
+import log from 'apify-shared/log';
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;
export const LOCAL_FILENAME_DIGITS = 9;
@@ -417,15 +418,17 @@ export class Dataset {
*
* @return {Promise}
*/
- delete() {
- return datasets
- .deleteDataset({
- datasetId: this.datasetId,
- })
- .then(() => {
- datasetsCache.remove(this.datasetId);
- if (this.datasetName) datasetsCache.remove(this.datasetName);
- });
+ async drop() {
+ await datasets.deleteDataset({ datasetId: this.datasetId });
+ datasetsCache.remove(this.datasetId);
+ if (this.datasetName) datasetsCache.remove(this.datasetName);
+ }
+
+ /** @ignore */
+ async delete() {
+ log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '
+ + 'This is to make it more obvious to users that the function deletes the dataset and not individual records in the dataset.');
+ await this.drop();
}
}
@@ -577,13 +580,17 @@ export class DatasetLocal {
return memo;
}
- delete() {
- return this.initializationPromise
- .then(() => emptyDirPromised(this.localStoragePath))
- .then(() => {
- this._updateMetadata(true);
- datasetsCache.remove(this.datasetId);
- });
+ async drop() {
+ await this.initializationPromise;
+ await emptyDirPromised(this.localStoragePath);
+ this._updateMetadata(true);
+ datasetsCache.remove(this.datasetId);
+ }
+
+ async delete() {
+ log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '
+ + 'This is to make it more obvious to users that the function deletes the dataset and not individual records in the dataset.');
+ await this.drop();
}
/**
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 76ae52fec..f757bc47b 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -7,6 +7,7 @@ import LruCache from 'apify-shared/lru_cache';
import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, KEY_VALUE_STORE_KEYS } from 'apify-shared/consts';
import { jsonStringifyExtended } from 'apify-shared/utilities';
+import log from 'apify-shared/log';
import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';
import {
addCharsetToContentType, apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists,
@@ -153,8 +154,8 @@ export const maybeStringify = (value, options) => {
* // text data returned as a string and other data is returned as binary buffer
* const value = await store.getValue('some-key');
*
- * // Delete record
- * await store.delete('some-key');
+ * // Drop (delete) the store
+ * await store.drop();
* ```
* @hideconstructor
*/
@@ -285,15 +286,17 @@ export class KeyValueStore {
*
* @return {Promise}
*/
- delete() {
- return keyValueStores
- .deleteStore({
- storeId: this.storeId,
- })
- .then(() => {
- storesCache.remove(this.storeId);
- if (this.storeName) storesCache.remove(this.storeName);
- });
+ async drop() {
+ await keyValueStores.deleteStore({ storeId: this.storeId });
+ storesCache.remove(this.storeId);
+ if (this.storeName) storesCache.remove(this.storeName);
+ }
+
+ /** @ignore */
+ async delete() {
+ log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '
+ + 'This is to make it more obvious to users that the function deletes the key-value store and not individual records in the store.');
+ await this.drop();
}
/**
@@ -421,6 +424,12 @@ export class KeyValueStoreLocal {
}
async delete() {
+ log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '
+ + 'This is to make it more obvious to users that the function deletes the key-value store and not individual records in the store.');
+ await this.drop();
+ }
+
+ async drop() {
await this.initializationPromise;
await emptyDirPromised(this.localStoragePath);
diff --git a/src/request_queue.js b/src/request_queue.js
index aca12d93b..d4f6ea47a 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -640,7 +640,7 @@ export class RequestQueue {
*
* @return {Promise}
*/
- async delete() {
+ async drop() {
await requestQueues.deleteQueue({
queueId: this.queueId,
});
@@ -649,6 +649,13 @@ export class RequestQueue {
if (this.queueName) queuesCache.remove(this.queueName);
}
+ /** @ignore */
+ async delete() {
+ log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '
+ + 'This is to make it more obvious to users that the function deletes the request queue and not individual records in the queue.');
+ await this.drop();
+ }
+
/**
* Returns the number of handled requests.
*
@@ -994,11 +1001,17 @@ export class RequestQueueLocal {
return this.pendingCount === 0;
}
- async delete() {
+ async drop() {
await emptyDirPromised(this.localStoragePath);
queuesCache.remove(this.queueId);
}
+ async delete() {
+ log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '
+ + 'This is to make it more obvious to users that the function deletes the request queue and not individual records in the queue.');
+ await this.drop();
+ }
+
async handledCount() {
const { handledRequestCount } = await this.getInfo();
return handledRequestCount;
diff --git a/test/dataset.js b/test/dataset.js
index 3f84ea83e..fb5e28d54 100644
--- a/test/dataset.js
+++ b/test/dataset.js
@@ -50,7 +50,7 @@ describe('dataset', () => {
// Delete works.
const datasetDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-dataset');
expectDirNonEmpty(datasetDir);
- await newDataset.delete();
+ await newDataset.drop();
expectDirEmpty(datasetDir);
});
@@ -344,7 +344,7 @@ describe('dataset', () => {
.once()
.withArgs({ datasetId: 'some-id' })
.returns(Promise.resolve());
- await dataset.delete();
+ await dataset.drop();
mock.verify();
mock.restore();
@@ -375,7 +375,7 @@ describe('dataset', () => {
.once()
.withArgs({ datasetId: 'some-id' })
.returns(Promise.resolve());
- await dataset.delete();
+ await dataset.drop();
mock.verify();
mock.restore();
@@ -408,7 +408,7 @@ describe('dataset', () => {
.once()
.withArgs({ datasetId: 'some-id' })
.returns(Promise.resolve());
- await dataset.delete();
+ await dataset.drop();
mock.verify();
mock.restore();
@@ -429,7 +429,7 @@ describe('dataset', () => {
.once()
.withArgs({ datasetId: 'some-id' })
.returns(Promise.resolve());
- await dataset.delete();
+ await dataset.drop();
mock.verify();
mock.restore();
});
@@ -454,7 +454,7 @@ describe('dataset', () => {
.once()
.withArgs({ datasetId: 'some-id' })
.returns(Promise.resolve());
- await dataset.delete();
+ await dataset.drop();
mock.verify();
mock.restore();
});
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 4b6493748..63b9e113f 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -96,7 +96,7 @@ describe('KeyValueStore', () => {
// Delete works.
const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-store-id');
expectDirNonEmpty(storeDir);
- await store.delete();
+ await store.drop();
expectDirEmpty(storeDir);
});
});
@@ -148,7 +148,7 @@ describe('KeyValueStore', () => {
storeId: 'some-id-1',
})
.returns(Promise.resolve());
- await store.delete();
+ await store.drop();
mock.verify();
mock.restore();
@@ -467,7 +467,7 @@ describe('KeyValueStore', () => {
// Delete works.
const storeDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, storeId);
expectDirNonEmpty(storeDir);
- await store.delete();
+ await store.drop();
expectDirEmpty(storeDir);
});
});
diff --git a/test/puppeteer_pool.js b/test/puppeteer_pool.js
index 3ab190699..d1be8cbf6 100644
--- a/test/puppeteer_pool.js
+++ b/test/puppeteer_pool.js
@@ -226,8 +226,6 @@ describe('PuppeteerPool', () => {
await pool.destroy();
});
- // Test started failing on 6.10.2018. Probably some change upstream.
- // Disabling the feature until resolved.
it('supports recycleDiskCache option', async () => {
const pool = new Apify.PuppeteerPool({
maxOpenPagesPerInstance: 1,
diff --git a/test/request_queue.js b/test/request_queue.js
index c9c33c142..538bba7e2 100644
--- a/test/request_queue.js
+++ b/test/request_queue.js
@@ -111,7 +111,7 @@ describe('RequestQueue', () => {
// Delete it.
const queueDir = path.join(LOCAL_STORAGE_DIR, LOCAL_STORAGE_SUBDIR, 'my-queue-0');
expectDirNonEmpty(queueDir);
- await queue.delete();
+ await queue.drop();
expectDirEmpty(queueDir);
});
@@ -457,7 +457,7 @@ describe('RequestQueue', () => {
queueId: 'some-id',
})
.returns(Promise.resolve());
- await queue.delete();
+ await queue.drop();
mock.verify();
mock.restore();