Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add key validation to KeyValueStore to match Platform implementation
Ondra Urban24ca7de06c72
diff --git a/src/key_value_store.js b/src/key_value_store.jsindex c83688717..2ae5ce433 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -4,6 +4,7 @@ import path from 'path';import Promise from 'bluebird';import contentTypeParser from 'content-type';import LruCache from 'apify-shared/lru_cache';+import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';import { ENV_VARS, LOCAL_EMULATION_SUBDIRS } from './constants';import { addCharsetToContentType, apifyClient, ensureDirExists } from './utils';@@ -55,6 +56,11 @@ const validateSetValueParams = (key, value, options) => {checkParamOrThrow(value, 'value', 'Buffer | String', 'The "value" parameter must be a String or Buffer when "options.contentType" is specified.'); // eslint-disable-line max-len}+ if (!KEY_VALUE_STORE_KEY_REGEX.test(key)) {+ throw new Error('The "key" parameter may contain only the following characters: ' ++ "[a-zA-Z0-9!-_.'()");+ }+if (options.contentType === '') throw new Error('Parameter options.contentType cannot be empty string.');if (!key) throw new Error('The "key" parameter cannot be empty');};diff --git a/test/key_value_store.js b/test/key_value_store.jsindex ee93cfc52..008c5874c 100644--- a/test/key_value_store.js+++ b/test/key_value_store.js@@ -73,6 +73,22 @@ describe('KeyValueStore', () => {await store.delete();expectDirEmpty(storeDir);});++ it('should throw on invalid keys', async () => {+ const store = new KeyValueStoreLocal('my-store-id', LOCAL_EMULATION_DIR);+ const INVALID_CHARACTERS = '?|\\/"*<>%:';+ let counter = 0;++ for (const char of INVALID_CHARACTERS) { // eslint-disable-line+ try {+ await store.setValue(`my_id_${char}`);+ } catch (e) {+ counter++;+ }+ }++ expect(counter).to.be.eql(INVALID_CHARACTERS.length);+ });});describe('remote', async () => {