Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Apify.getValue() and setValue() now can use local dir if used with the APIFY_DEV_KEY_VALUE_STORE_DIR env variable to simplify local development, added unit tests
Jan Curn07d111cdf9b8
diff --git a/package.json b/package.jsonindex 4e26834cb..27259eb77 100644--- a/package.json+++ b/package.json@@ -43,7 +43,7 @@"apify-client": ">=0.1.13","underscore": "^1.8.3","portastic": "^1.0.1",- "tmp": "^0.0.31",+ "tmp": "^0.0.33","bluebird": "^3.5.0","rimraf": "^2.6.1"},diff --git a/src/actor.js b/src/actor.jsindex 4bb7afe08..39d702784 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,9 +1,12 @@import fs from 'fs';+import path from 'path';import _ from 'underscore';+import Promise from 'bluebird';import { checkParamOrThrow } from 'apify-client/build/utils';import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES } from './constants';import { getPromisePrototype, newPromise, nodeifyPromise, newClient } from './utils';+/* global process, Buffer *//**@@ -12,6 +15,10 @@ import { getPromisePrototype, newPromise, nodeifyPromise, newClient } from './ut*/export const apifyClient = newClient();+const readFilePromised = Promise.promisify(fs.readFile);+const writeFilePromised = Promise.promisify(fs.writeFile);+const unlinkPromised = Promise.promisify(fs.unlink);+/*** Tries to parse a string with date.* @param str Date string@@ -37,24 +44,58 @@ const getDefaultStoreIdOrThrow = () => {* content type, the body is the already parsed object. For other content types,* the body is raw String or Buffer. If the record cannot be found, the result is null.* or `null` if record was not found.+ *+ * If the `APIFY_DEV_KEY_VALUE_STORE_DIR` environment variable is defined,+ * the value is read from a that directory rather than the key-value store,+ * from a file that has the key as a name.+ * The file is assumed to have a content type specified in the `APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE`+ * environment variable, or `application/json` if not set.+ * This is useful for local development of the act.* @param callback Optional callback.* @return Returns a promise if no callback was provided, otherwise the return value is not defined.*/export const getValue = (key, callback = null) => {if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');- const storeId = getDefaultStoreIdOrThrow();- const promisePrototype = getPromisePrototype();+ const devDir = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_DIR];+ let promise;- const promise = newPromise()- .then(() => {- return apifyClient.keyValueStores.getRecord({- storeId,- promise: promisePrototype,- key,- rawBody: true,+ if (devDir) {+ // We're emulating KV store locally in a directory to simplify development+ const contentType = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_CONTENT_TYPE] || 'application/json';++ const filePath = path.join(devDir, key);+ promise = newPromise()+ .then(() => {+ return readFilePromised(filePath);+ })+ .then((data) => {+ // Parse JSON+ if (contentType === 'application/json') {+ try {+ data = JSON.parse(data.toString('utf8'));+ } catch (e) {+ throw new Error(`File at '${filePath}' cannot be parsed as JSON: ${e.message}`);+ }+ } else if (contentType === 'text/plain') {+ data = data.toString();+ }+ return data;});- });+ } else {+ const storeId = getDefaultStoreIdOrThrow();+ const promisePrototype = getPromisePrototype();++ promise = newPromise()+ .then(() => {+ return apifyClient.keyValueStores.getRecord({+ storeId,+ promise: promisePrototype,+ key,+ rawBody: true,+ });+ });+ }return nodeifyPromise(promise, callback);};@@ -64,10 +105,14 @@ export const getValue = (key, callback = null) => {* This data is stored in the key-value store created specifically for this run,* whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.* The function has no result, but throws on invalid args or other errors.+ *+ * If the `APIFY_DEV_KEY_VALUE_STORE_DIR` environment variable is defined,+ * the value is written to that directory rather than the key-value store,+ * to a file named as the key. This is useful for local development of the act.* @param value* If null, the record in the key-value store is deleted.* If no contentType is specified, the value can be any object and it will be stringified to JSON.- * If contentType is specified, value is considered raw data it must be a String or Buffer.+ * If contentType is specified, value is considered raw data and it must be a String or Buffer.* For any other value an error will be thrown.* @param options Optional settings, currently only { contentType: String } is supported to set MIME content type of the value.* @param callback Optional callback.@@ -89,6 +134,8 @@ export const setValue = (key, value, options, callback = null) => {const storeId = getDefaultStoreIdOrThrow();const promisePrototype = getPromisePrototype();+ const devDir = process.env[ENV_VARS.DEV_KEY_VALUE_STORE_DIR];+let innerPromise;if (value !== null) {// Normal case: put record to store@@ -113,24 +160,36 @@ export const setValue = (key, value, options, callback = null) => {throw new Error('The "value" parameter must be a String or Buffer when "contentType" is specified.');}- // Keep this code in main scope so that simple errors are thrown rather than rejected promise.- innerPromise = apifyClient.keyValueStores.putRecord({- storeId,- promise: promisePrototype,- key,- body: value,- contentType: options.contentType,- });+ if (devDir) {+ // We're emulating KV store locally in a directory to simplify development+ const filePath = path.join(devDir, key);+ innerPromise = writeFilePromised(filePath, value);+ } else {+ // Keep this code in main scope so that simple errors are thrown rather than rejected promise.+ innerPromise = apifyClient.keyValueStores.putRecord({+ storeId,+ promise: promisePrototype,+ key,+ body: value,+ contentType: options.contentType,+ });+ }} else {// Special case: remove the record from the storeif (options.contentType !== null && options.contentType !== undefined) {throw new Error('The "options.contentType" parameter must not be used when removing the record.');}- innerPromise = apifyClient.keyValueStores.deleteRecord({- storeId,- promise: promisePrototype,- key,- });+ if (devDir) {+ // We're emulating KV store locally in a directory to simplify development+ const filePath = path.join(devDir, key);+ innerPromise = unlinkPromised(filePath);+ } else {+ innerPromise = apifyClient.keyValueStores.deleteRecord({+ storeId,+ promise: promisePrototype,+ key,+ });+ }}const promise = newPromise().then(() => innerPromise);diff --git a/src/constants.js b/src/constants.jsindex 976ef1039..cb331887e 100644--- a/src/constants.js+++ b/src/constants.js@@ -21,6 +21,8 @@ export const ENV_VARS = {STARTED_AT: 'APIFY_STARTED_AT',TIMEOUT_AT: 'APIFY_TIMEOUT_AT',DEFAULT_KEY_VALUE_STORE_ID: 'APIFY_DEFAULT_KEY_VALUE_STORE_ID',+ DEV_KEY_VALUE_STORE_DIR: 'APIFY_DEV_KEY_VALUE_STORE_DIR',+ DEV_KEY_VALUE_STORE_CONTENT_TYPE: 'APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE',WATCH_FILE: 'APIFY_WATCH_FILE',API_BASE_URL: 'APIFY_API_BASE_URL',HEADLESS: 'APIFY_HEADLESS',diff --git a/test/actor.js b/test/actor.jsindex 3f9a4f2f0..452570c62 100644--- a/test/actor.js+++ b/test/actor.js@@ -1,8 +1,10 @@import fs from 'fs';+import pathModule from 'path';import _ from 'underscore';import { expect } from 'chai';import sinon from 'sinon';import tmp from 'tmp';+import rimraf from 'rimraf';import Promise from 'bluebird';import { ACT_TASK_STATUSES } from '../build/constants';@@ -155,11 +157,11 @@ const testMain = ({ userFunc, exitCode }) => {// Call user func to test other behavior (note that it can throw)if (userFunc) return userFunc();});+ })+ .catch((err) => {+ error = err;});})- .catch((err) => {- error = err;- }).then(() => {// Waits max 1000 millis for process.exit() mock to be called// console.log(`XXX: grand finale: ${err}`);@@ -345,7 +347,7 @@ describe('Apify.getValue()', () => {.then(() => {// test callbackreturn new Promise((resolve, reject) => {- return Apify.getValue('INPUT', (err, input) => {+ Apify.getValue('INPUT', (err, input) => {if (err) return reject(err);resolve(input);});@@ -371,23 +373,29 @@ describe('Apify.getValue()', () => {return Promise.resolve().then(() => {// test promise- return Apify.getValue('INPUT');- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');+ return Apify.getValue('INPUT')+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.be.eql('Test error');+ });}).then(() => {// test callbackreturn new Promise((resolve, reject) => {- return Apify.getValue('INPUT', (err, input) => {+ Apify.getValue('INPUT', (err, input) => {if (err) return reject(err);resolve(input);});+ })+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.be.eql('Test error');});})- .catch((err) => {- expect(err.message).to.be.eql('Test error');- }).then(() => {mock.verify();})@@ -395,6 +403,108 @@ describe('Apify.getValue()', () => {mock.restore();});});++ it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on success)', () => {+ const jsonValue = { test: 123 };+ const textValue = 'some text \u00e6\u00f8\u00e5';+ const rawValue = Buffer.from('bla bla bla');+ const tmpobj = tmp.dirSync();+ process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;++ return Promise.resolve()+ .then(() => {+ // Test JSON with default content type+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;+ fs.writeFileSync(pathModule.join(tmpobj.name, 'INPUT'), JSON.stringify(jsonValue));+ return Apify.getValue('INPUT');+ })+ .then((value) => {+ expect(value).to.be.eql(jsonValue);+ })+ .then(() => {+ // Test text+ fs.writeFileSync(pathModule.join(tmpobj.name, 'TEST_TEXT'), textValue);+ process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'text/plain';+ return Apify.getValue('TEST_TEXT');+ })+ .then((value) => {+ expect(value).to.be.eql(textValue);+ })+ .then(() => {+ // Test raw data+ fs.writeFileSync(pathModule.join(tmpobj.name, 'TEST_RAW'), rawValue);+ process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'something/else';+ return Apify.getValue('TEST_RAW');+ })+ .then((value) => {+ expect(value).to.be.eql(rawValue);+ })+ .then(() => {+ // Test callback with JSON plus explicit content type+ process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'application/json';+ return new Promise((resolve, reject) => {+ Apify.getValue('INPUT', (err, input) => {+ if (err) return reject(err);+ resolve(input);+ });+ });+ })+ .then((value) => {+ expect(value).to.be.eql(jsonValue);+ })+ .finally(() => {+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;+ rimraf.sync(tmpobj.name);+ });+ });++ it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on error)', () => {+ const tmpobj = tmp.dirSync();+ process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;++ return Promise.resolve()+ .then(() => {+ // Test invalid JSON+ fs.writeFileSync(pathModule.join(tmpobj.name, 'INPUT'), 'something not JSON');+ return Apify.getValue('INPUT')+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.contain('cannot be parsed as JSON');+ });+ })+ .then(() => {+ // Test non-existent file+ return Apify.getValue('NON_EXISTENT_KEY')+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.contain('ENOENT');+ });+ })+ .then(() => {+ // Test callback plus non-existent directory+ process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = pathModule.join(tmpobj.name, '/blabla/');+ return new Promise((resolve, reject) => {+ Apify.getValue('INPUT', (err, input) => {+ if (err) return reject(err);+ resolve(input);+ });+ })+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.contain('ENOENT');+ });+ })+ .finally(() => {+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;+ });+ });});describe('Apify.setValue()', () => {@@ -474,7 +584,7 @@ describe('Apify.setValue()', () => {.then(() => {// test callback (no options)return new Promise((resolve, reject) => {- return Apify.setValue('mykey', { someValue: 123 }, (err) => {+ Apify.setValue('mykey', { someValue: 123 }, (err) => {if (err) return reject(err);resolve();});@@ -483,7 +593,7 @@ describe('Apify.setValue()', () => {.then(() => {// test callback (with options)return new Promise((resolve, reject) => {- return Apify.setValue('mykey', 'myvalue', { contentType: 'text/plain' }, (err) => {+ Apify.setValue('mykey', 'myvalue', { contentType: 'text/plain' }, (err) => {if (err) return reject(err);resolve();});@@ -492,7 +602,7 @@ describe('Apify.setValue()', () => {.then(() => {// test callback (null options)return new Promise((resolve, reject) => {- return Apify.setValue('mykey', { someValue: 123 }, null, (err) => {+ Apify.setValue('mykey', { someValue: 123 }, null, (err) => {if (err) return reject(err);resolve();});@@ -501,7 +611,7 @@ describe('Apify.setValue()', () => {.then(() => {// test callback (undefined options)return new Promise((resolve, reject) => {- return Apify.setValue('mykey', { someValue: 123 }, resolve, (err) => {+ Apify.setValue('mykey', { someValue: 123 }, resolve, (err) => {if (err) return reject(err);resolve();});@@ -524,24 +634,33 @@ describe('Apify.setValue()', () => {return Promise.resolve().then(() => {- // test promise- return Apify.setValue('mykey', { someValue: 1 });- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');+ // Test promise+ return Promise.resolve()+ .then(() => {+ return Apify.setValue('mykey', { someValue: 1 });+ })+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.be.eql('Test error');+ });}).then(() => {- // test callback+ // Test callbackreturn new Promise((resolve, reject) => {- return Apify.setValue('mykey', { someValue: 1 }, (err) => {+ Apify.setValue('mykey', { someValue: 1 }, (err) => {if (err) return reject(err);resolve();});+ })+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.be.eql('Test error');});})- .catch((err) => {- expect(err.message).to.be.eql('Test error');- }).then(() => {mock.verify();})@@ -653,6 +772,105 @@ describe('Apify.setValue()', () => {mock.restore();});});++ it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on success)', () => {+ const tmpobj = tmp.dirSync();+ const testObj = { 'bla-bla': 123 };+ const valueString = 'bla bla some string';+ const valueRaw = Buffer.from('some other string that will be raw as ham');+ process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = tmpobj.name;++ return Promise.resolve()+ .then(() => {+ // Test write object+ Apify.setValue('TEST_OBJ', testObj);+ })+ .then(() => {+ return Apify.getValue('TEST_OBJ');+ })+ .then((value) => {+ expect(value).to.be.eql(testObj);+ })+ .then(() => {+ // Test removal of value+ Apify.setValue('TEST_OBJ', null);+ })+ .then(() => {+ // File must no longer exists+ const filePath = pathModule.join(tmpobj.name, '/TEST_OBJ');+ const exists = fs.existsSync(filePath);+ expect(exists).to.be.eql(false);+ })+ .then(() => {+ // Test write of string with callbacks+ return new Promise((resolve, reject) => {+ Apify.setValue('TEST_STR', valueString, { contentType: 'text/plain' }, (err, input) => {+ if (err) return reject(err);+ resolve(input);+ });+ });+ })+ .then(() => {+ process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'text/plain';+ return Apify.getValue('TEST_STR');+ })+ .then((value) => {+ expect(value).to.be.eql(valueString);+ })+ .then(() => {+ // Test write raw buffer+ Apify.setValue('TEST_RAW', valueRaw, { contentType: 'something/whatever' });+ })+ .then(() => {+ process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE = 'something/raw';+ return Apify.getValue('TEST_RAW');+ })+ .then((value) => {+ expect(value).to.be.eql(valueRaw);+ })+ .finally(() => {+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;+ // rimraf.sync(tmpobj.name);+ });+ });++ it('works with APIFY_DEV_KEY_VALUE_STORE_DIR env var (on error)', () => {+ const tmpobj = tmp.dirSync();+ process.env.APIFY_DEV_KEY_VALUE_STORE_DIR = pathModule.join(tmpobj.name, '/non-existent-dir/');++ return Promise.resolve()+ .then(() => {+ // Test write object with Promise+ Apify.setValue('TEST1', { whatever: 123 })+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.contain('ENOENT');+ });+ })+ .then(() => {+ // Test callback plus non-existent directory+ return new Promise((resolve, reject) => {+ Apify.setValue('TEST2', { sometingElse: 456 }, (err, input) => {+ if (err) return reject(err);+ resolve(input);+ });+ })+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.contain('ENOENT');+ });+ })+ .finally(() => {+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;+ delete process.env.APIFY_DEV_KEY_VALUE_STORE_CONTENT_TYPE;+ // rimraf.sync(tmpobj.name);+ });+ });});describe('Apify.events', () => {