Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixing storage.remove() that doesnt remove storage from cache when saved by name

Marek Trunkate1a0e5bbbce0
diff --git a/src/dataset.js b/src/dataset.js
index aae17f2d5..333eff1ec 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -150,10 +150,12 @@ export const chunkBySize = (items, limitBytes) => {
* @hideconstructor
*/
export class Dataset {
- constructor(datasetId) {
+ constructor(datasetId, datasetName) {
checkParamOrThrow(datasetId, 'datasetId', 'String');
+ checkParamOrThrow(datasetName, 'datasetName', 'Maybe String');
this.datasetId = datasetId;
+ this.datasetName = datasetName;
}
/**
@@ -408,6 +410,7 @@ export class Dataset {
})
.then(() => {
datasetsCache.remove(this.datasetId);
+ datasetsCache.remove(this.datasetName);
});
}
}
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 1f31930e4..e0d8a3470 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -130,10 +130,12 @@ export const maybeStringify = (value, options) => {
* @hideconstructor
*/
export class KeyValueStore {
- constructor(storeId) {
+ constructor(storeId, storeName) {
checkParamOrThrow(storeId, 'storeId', 'String');
+ checkParamOrThrow(storeName, 'storeName', 'Maybe String');
this.storeId = storeId;
+ this.storeName = storeName;
}
// TODO: Move here the Apify.getValue()/setValue() documentation, and link it from there.
@@ -197,6 +199,7 @@ export class KeyValueStore {
})
.then(() => {
storesCache.remove(this.storeId);
+ storesCache.remove(this.storeName);
});
}
}
diff --git a/src/request_queue.js b/src/request_queue.js
index 5280c3731..2e2a00030 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -179,10 +179,12 @@ const getRequestId = (uniqueKey) => {
* @hideconstructor
*/
export class RequestQueue {
- constructor(queueId) {
+ constructor(queueId, queueName) {
checkParamOrThrow(queueId, 'queueId', 'String');
+ checkParamOrThrow(queueName, 'queueName', 'Maybe String');
this.queueId = queueId;
+ this.queueName = queueName;
this.queueHeadDict = new ListDictionary();
this.requestIdsInProgress = {};
this.inProgressCount = 0;
@@ -513,6 +515,7 @@ export class RequestQueue {
})
.then(() => {
queuesCache.remove(this.queueId);
+ queuesCache.remove(this.queueName);
});
}
}
diff --git a/src/utils.js b/src/utils.js
index 06a972781..474e94977 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -441,7 +441,7 @@ export const openRemoteStorage = async (idOrName, defaultIdEnvVar, RemoteClass,
if (!storagePromise) {
storagePromise = isDefault // If true then we know that this is an ID of existing store.
? Promise.resolve(new RemoteClass(idOrName))
- : getOrCreateFunction(idOrName).then(storage => (new RemoteClass(storage.id)));
+ : getOrCreateFunction(idOrName).then(storage => (new RemoteClass(storage.id, storage.name)));
cache.add(idOrName, storagePromise);
}