Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Bugfix in Apify.setValue() - APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not always required, Apify.getValue() also supports 'application/json; charset=utf-8' content type, improved unit tests
Jan Curn2ee306de9b0f
diff --git a/package.json b/package.jsonindex 1a349e08b..6c8d17b87 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.4.21",+ "version": "0.4.22","description": "Apify runtime for JavaScript","main": "build/index.js","keywords": [diff --git a/src/actor.js b/src/actor.jsindex 898e87324..7f12f8363 100644--- a/src/actor.js+++ b/src/actor.js@@ -91,7 +91,7 @@ export const getValue = (key, callback = null) => {.then((data) => {// Parse file according to the content typeif (data !== null) {- if (contentType === 'application/json') {+ if (contentType === 'application/json' || contentType === 'application/json; charset=utf-8') {try {data = JSON.parse(data.toString('utf8'));} catch (e) {@@ -155,9 +155,10 @@ export const setValue = (key, value, options, callback = null) => {// Make copy of options, don't update what user passedoptions = Object.assign({}, options);- const storeId = getDefaultStoreIdOrThrow();const promisePrototype = getPromisePrototype();+ let storeId = null;+// Handle emulation of KV store locally in a directory to simplify developmentconst devDir = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_DIR];let devDirPath;@@ -166,6 +167,9 @@ export const setValue = (key, value, options, callback = null) => {// Get absolute pathsdevDirPath = path.resolve(devDir);devFilePath = path.resolve(devDirPath, key);+ } else {+ // This would throw if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var was not set+ storeId = getDefaultStoreIdOrThrow();}const devErrorHandler = (err) => {throw new Error(`Error writing file '${key}' in directory '${devDirPath}' referred by ${ENV_VARS.DEV_KEY_VALUE_STORE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-lendiff --git a/test/actor.js b/test/actor.jsindex b2d951589..be0d1b317 100644--- a/test/actor.js+++ b/test/actor.js@@ -411,6 +411,9 @@ describe('Apify.getValue()', () => {const tmpobj = tmp.dirSync();process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;+ // Test that this env var doesn't need to be set+ delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;+return Promise.resolve().then(() => {// Test JSON with default content type@@ -431,6 +434,9 @@ describe('Apify.getValue()', () => {expect(value).to.be.eql(textValue);}).then(() => {+ // Test that this env var can be set to anything+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = 'whatever';+// Test raw datafs.writeFileSync(pathModule.join(tmpobj.name, 'TEST_RAW'), rawValue);process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'something/else';@@ -448,7 +454,7 @@ describe('Apify.getValue()', () => {}).then(() => {// Test callback with JSON plus explicit content type- process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'application/json';+ process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'application/json; charset=utf-8';return new Promise((resolve, reject) => {Apify.getValue('INPUT', (err, input) => {if (err) return reject(err);@@ -790,6 +796,9 @@ describe('Apify.setValue()', () => {const valueRaw = Buffer.from('some other string that will be raw as ham');process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;+ // Test that this env var doesn't need to be set+ delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;+return Promise.resolve().then(() => {// Test write object@@ -812,6 +821,9 @@ describe('Apify.setValue()', () => {expect(exists).to.be.eql(false);}).then(() => {+ // Test that this env var can be set to anything+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = 'whatever';+// Test write of string with callbacksreturn new Promise((resolve, reject) => {Apify.setValue('TEST_STR', valueString, { contentType: 'text/plain' }, (err, input) => {