Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fixing KV store bugs causing unhandled promise rejection

Marek Trunkata88f4aad9d8e
diff --git a/src/key_value_store.js b/src/key_value_store.js
index cee00b3ea..43a066679 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -355,31 +355,38 @@ export class KeyValueStoreLocal {
this.initializationPromise = ensureDirExists(this.localStoragePath);
}
- getValue(key) {
+ async getValue(key) {
validateGetValueParams(key);
- return this.initializationPromise
- .then(() => this._handleFile(key, readFilePromised))
- .then((result) => {
- return result
- ? parseBody(result.returnValue, mime.getType(result.fileName))
- : null;
- })
- .catch((err) => {
- throw new Error(`Error reading file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
- });
+ await this.initializationPromise;
+
+ try {
+ const result = await this._handleFile(key, readFilePromised);
+ return result
+ ? parseBody(result.returnValue, mime.getType(result.fileName))
+ : null;
+ } catch (err) {
+ throw new Error(`Error reading file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line
+ }
}
- setValue(key, value, options = {}) {
+ async setValue(key, value, options = {}) {
validateSetValueParams(key, value, options);
+ await this.initializationPromise;
+
// Make copy of options, don't update what user passed.
const optionsCopy = Object.assign({}, options);
- const deletePromise = this._handleFile(key, unlinkPromised);
+ // First remove original file.
+ try {
+ await this._handleFile(key, unlinkPromised);
+ } catch (err) {
+ throw new Error(`Error removing file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
+ }
- // In this case delete the record.
- if (value === null) return deletePromise;
+ // In this case just delete the record.
+ if (value === null) return;
value = maybeStringify(value, optionsCopy);
@@ -387,18 +394,18 @@ export class KeyValueStoreLocal {
const extension = mime.getExtension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
const filePath = this._getPath(`${key}.${extension}`);
- return deletePromise
- .then(() => writeFilePromised(filePath, value))
- .catch((err) => {
- throw new Error(`Error writing file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
- });
+ try {
+ await writeFilePromised(filePath, value);
+ } catch (err) {
+ throw new Error(`Error writing file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
+ }
}
- delete() {
- return emptyDirPromised(this.localStoragePath)
- .then(() => {
- storesCache.remove(this.storeId);
- });
+ async delete() {
+ await this.initializationPromise;
+ await emptyDirPromised(this.localStoragePath);
+
+ storesCache.remove(this.storeId);
}
async forEachKey(iteratee, options = {}, index = 0) {
@@ -407,6 +414,8 @@ export class KeyValueStoreLocal {
checkParamOrThrow(exclusiveStartKey, 'options.exclusiveStartKey', 'Maybe String');
checkParamOrThrow(index, 'index', 'Number');
+ await this.initializationPromise;
+
const files = await readdirPromised(this.localStoragePath);
let keys = files.map(file => path.parse(file).name).sort(); // Array is sorted to emulate API.
if (exclusiveStartKey) {
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 71c8e8640..ba71db032 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -212,6 +212,7 @@ describe('KeyValueStore', () => {
it('throws on invalid args', async () => {
process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] = '12345';
process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
+
await expect(Apify.setValue()).to.be.rejectedWith('Parameter "key" of type String must be provided');
await expect(Apify.setValue('', null)).to.be.rejectedWith('The "key" parameter cannot be empty');
await expect(Apify.setValue('', 'some value')).to.be.rejectedWith('The "key" parameter cannot be empty');