Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add key validation to KeyValueStore to match Platform implementation

Ondra Urban24ca7de06c72
diff --git a/src/key_value_store.js b/src/key_value_store.js
index c83688717..2ae5ce433 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -4,6 +4,7 @@ import path from 'path';
import Promise from 'bluebird';
import contentTypeParser from 'content-type';
import LruCache from 'apify-shared/lru_cache';
+import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';
import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_EMULATION_SUBDIRS } from './constants';
import { addCharsetToContentType, apifyClient, ensureDirExists } from './utils';
@@ -55,6 +56,11 @@ const validateSetValueParams = (key, value, options) => {
checkParamOrThrow(value, 'value', 'Buffer | String', 'The "value" parameter must be a String or Buffer when "options.contentType" is specified.'); // eslint-disable-line max-len
}
+ if (!KEY_VALUE_STORE_KEY_REGEX.test(key)) {
+ throw new Error('The "key" parameter may contain only the following characters: ' +
+ "[a-zA-Z0-9!-_.'()");
+ }
+
if (options.contentType === '') throw new Error('Parameter options.contentType cannot be empty string.');
if (!key) throw new Error('The "key" parameter cannot be empty');
};
diff --git a/test/key_value_store.js b/test/key_value_store.js
index ee93cfc52..008c5874c 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -73,6 +73,22 @@ describe('KeyValueStore', () => {
await store.delete();
expectDirEmpty(storeDir);
});
+
+ it('should throw on invalid keys', async () => {
+ const store = new KeyValueStoreLocal('my-store-id', LOCAL_EMULATION_DIR);
+ const INVALID_CHARACTERS = '?|\\/"*<>%:';
+ let counter = 0;
+
+ for (const char of INVALID_CHARACTERS) { // eslint-disable-line
+ try {
+ await store.setValue(`my_id_${char}`);
+ } catch (e) {
+ counter++;
+ }
+ }
+
+ expect(counter).to.be.eql(INVALID_CHARACTERS.length);
+ });
});
describe('remote', async () => {