Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Removed Apify.getInput() and Apify.setOutput(), changed Apify.getContext() and Apify.main() semantics not to handle input/output implicitly to make system more transparent

Jan Curn39f60819f3ad
diff --git a/README.md b/README.md
index 19225d46a..6602b907f 100644
--- a/README.md
+++ b/README.md
@@ -263,7 +263,7 @@ await browser.close();
### Promises
-By default, the `getContext`, `getValue`, `setValue` and `browse` functions return a promise.
+By default, the `getValue`, `setValue` and `browse` functions return a promise.
However, they also accept a Node.js-style callback parameter.
If the callback is provided, the return value of the functions is not defined
and the functions only invoke the callback upon completion or error.
diff --git a/src/actor.js b/src/actor.js
index 3fff7aa04..173502f80 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,6 +1,6 @@
import fs from 'fs';
import _ from 'underscore';
-import { ENV_VARS, EXIT_CODES, KEY_VALUE_STORE_KEYS } from './constants';
+import { ENV_VARS, EXIT_CODES } from './constants';
import { getPromisePrototype, newPromise, nodeifyPromise, newClient } from './utils';
/* global process, Buffer */
@@ -36,127 +36,51 @@ const getDefaultStoreIdOrThrow = () => {
/**
- * Gets input data for the current act run.
- * This data is stored in the key-value store created specifically for this run,
- * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable,
- * under record with the `INPUT` key.
- * The result of the function is an object such as `{ body: String|Buffer, contentType: String|null }`,
+ * Gets a value from the default key-value store for the current act run.
+ * This store is created automatically for this run,
+ * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
+ * The result of the function is the body of the record. For records with 'application/json'
+ * content type, the body is the already parsed object. For other content types,
+ * the body is raw String or Buffer. If the record cannot be found, the result is null.
* or `null` if record was not found.
* @param callback Optional callback.
* @return Returns a promise if no callback was provided, otherwise the return value is not defined.
*/
-export const getInput = (callback = null) => {
- // TODO: get rid of this
+export const getValue = (key, callback = null) => {
+ if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');
+
+ const storeId = getDefaultStoreIdOrThrow();
+ const promisePrototype = getPromisePrototype();
+
const promise = newPromise()
.then(() => {
return apifyClient.keyValueStores.getRecord({
- storeId: getDefaultStoreIdOrThrow(),
- promise: getPromisePrototype(),
- key: KEY_VALUE_STORE_KEYS.INPUT,
+ storeId,
+ promise: promisePrototype,
+ key,
});
})
- .then((input) => {
- // Ensure we always return either:
+ .then((record) => {
+ // Check that the record is always either:
// * null
// * or { body: String|Buffer, contentType: String|null }
// * or { body: Any, contentType: 'application/json' }
const baseMsg = 'ApifyClient returned an unexpected value from keyValueStores.getRecord()';
- if (!input) {
- input = null;
- } else if (typeof (input) !== 'object') {
+ if (!record) {
+ return null;
+ }
+
+ if (typeof (record) !== 'object') {
throw new Error(`${baseMsg}: expected an object.`);
- } else if ((typeof (input.contentType) !== 'string' && input.contentType !== null)) {
+ } else if ((typeof (record.contentType) !== 'string' && record.contentType !== null)) {
throw new Error(`${baseMsg}: contentType is not valid.`);
- } else if (!_.contains(JSON_CONTENT_TYPES, input.contentType)
- && typeof (input.body) !== 'string'
- && !Buffer.isBuffer(input.body)) {
+ } else if (!_.contains(JSON_CONTENT_TYPES, record.contentType)
+ && typeof (record.body) !== 'string'
+ && !Buffer.isBuffer(record.body)) {
throw new Error(`${baseMsg}: body must be String or Buffer.`);
}
- return input;
- });
-
- return nodeifyPromise(promise, callback);
-};
-
-/**
- * Sets output data for the current act run.
- * This data is stored in the key-value store created specifically for this run,
- * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable,
- * under record with the `OUTPUT` key.
- * The function has no result, but throws on invalid args.
- * @param output Must be an object such as { body: String|Buffer, contentType: String|null }
- * @param callback Optional callback.
- * @return Returns a promise if no callback was provided, otherwise the return value is not defined.
- */
-export const setOutput = (output, callback = null) => {
- if (typeof (output) !== 'object') {
- throw new Error('The "output" parameter must be an object.');
- }
- if (typeof (output.body) !== 'string' && !Buffer.isBuffer(output.body)) {
- throw new Error('The "output.body" parameter must be String or Buffer.');
- }
- if (typeof (output.contentType) !== 'string' && output.contentType !== null) {
- throw new Error('The "output.contentType" parameter must be String or null.');
- }
-
- const promise = newPromise()
- .then(() => {
- return apifyClient.keyValueStores.putRecord({
- storeId: getDefaultStoreIdOrThrow(),
- promise: getPromisePrototype(),
- key: KEY_VALUE_STORE_KEYS.OUTPUT,
- body: output.body,
- contentType: output.contentType,
- });
- });
-
- return nodeifyPromise(promise, callback);
-};
-
-
-/**
- * Gets a value from the default key-value store for the current act run.
- * This data is stored in the key-value store created specifically for this run,
- * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.
- * The result of the function is the body of the record. For records with 'application/json'
- * content type, the body is the already parsed object. For other content types,
- * the body is raw String or Buffer. If the record cannot be found, the result is null.
- * or `null` if record was not found.
- * @param callback Optional callback.
- * @return Returns a promise if no callback was provided, otherwise the return value is not defined.
- */
-export const getValue = (key, callback = null) => {
- if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');
-
- const promise = newPromise()
- .then(() => {
- return apifyClient.keyValueStores.getRecord({
- storeId: getDefaultStoreIdOrThrow(),
- promise: getPromisePrototype(),
- key,
+ return record.body;
});
- })
- .then((record) => {
- // Check that the record is always either:
- // * null
- // * or { body: String|Buffer, contentType: String|null }
- // * or { body: Any, contentType: 'application/json' }
- const baseMsg = 'ApifyClient returned an unexpected value from keyValueStores.getRecord()';
- if (!record) {
- return null;
- }
-
- if (typeof (record) !== 'object') {
- throw new Error(`${baseMsg}: expected an object.`);
- } else if ((typeof (record.contentType) !== 'string' && record.contentType !== null)) {
- throw new Error(`${baseMsg}: contentType is not valid.`);
- } else if (!_.contains(JSON_CONTENT_TYPES, record.contentType)
- && typeof (record.body) !== 'string'
- && !Buffer.isBuffer(record.body)) {
- throw new Error(`${baseMsg}: body must be String or Buffer.`);
- }
- return record.body;
- });
return nodeifyPromise(promise, callback);
};
@@ -184,6 +108,9 @@ export const setValue = (key, value, contentType, callback = null) => {
contentType = null;
}
+ const storeId = getDefaultStoreIdOrThrow();
+ const promisePrototype = getPromisePrototype();
+
let innerPromise;
if (value !== null) {
// Normal case: put record to store
@@ -209,8 +136,8 @@ export const setValue = (key, value, contentType, callback = null) => {
// Keep this code in main scope so that simple errors are thrown rather than rejected promise.
innerPromise = apifyClient.keyValueStores.putRecord({
- storeId: getDefaultStoreIdOrThrow(),
- promise: getPromisePrototype(),
+ storeId,
+ promise: promisePrototype,
key,
body: value,
contentType,
@@ -222,8 +149,8 @@ export const setValue = (key, value, contentType, callback = null) => {
throw new Error('The "contentType" parameter must not be used when removing the record.');
}
innerPromise = apifyClient.keyValueStores.deleteRecord({
- storeId: getDefaultStoreIdOrThrow(),
- promise: getPromisePrototype(),
+ storeId,
+ promise: promisePrototype,
key,
});
}
@@ -237,48 +164,33 @@ export const setValue = (key, value, contentType, callback = null) => {
* Generates a context object which contains meta-data about this act run such as:
* ```javascript
* {
- * internalPort: Number,
* actId: String,
* actRunId: String,
+ * userId: String,
+ * token: String,
* startedAt: Date,
* timeoutAt: Date,
* defaultKeyValueStoreId: String,
- * input: {
- * body: String/Buffer,
- * contentType: String,
- * }
+ * internalPort: Number,
* }
* ```
- * The information is generate using the APIFY_XXX environment variables and the input data is fetched from Apify API.
+ * All the information is generated from the APIFY_XXX environment variables.
* If some of the variables is not defined or is invalid, the corresponding value in the context object will be null;
* an error is not thrown in such a case in order to simplify local development and debugging of acts.
- * @param callback Optional callback.
- * @return Returns a promise if no callback was provided, otherwise the return value is not defined.
+ * @return Object Context object.
*/
-export const getContext = (callback = null) => {
- const context = {
- internalPort: parseInt(process.env[ENV_VARS.INTERNAL_PORT], 10) || null,
- actId: process.env[ENV_VARS.ACT_ID] || null,
- actRunId: process.env[ENV_VARS.ACT_RUN_ID] || null,
- userId: process.env[ENV_VARS.USER_ID] || null,
- token: process.env[ENV_VARS.TOKEN] || null,
- startedAt: tryParseDate(process.env[ENV_VARS.STARTED_AT]) || null,
- timeoutAt: tryParseDate(process.env[ENV_VARS.TIMEOUT_AT]) || null,
- defaultKeyValueStoreId: process.env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] || null,
- input: null,
+export const getContext = () => {
+ const env = process.env || {};
+ return {
+ actId: env[ENV_VARS.ACT_ID] || null,
+ actRunId: env[ENV_VARS.ACT_RUN_ID] || null,
+ userId: env[ENV_VARS.USER_ID] || null,
+ token: env[ENV_VARS.TOKEN] || null,
+ startedAt: tryParseDate(env[ENV_VARS.STARTED_AT]) || null,
+ timeoutAt: tryParseDate(env[ENV_VARS.TIMEOUT_AT]) || null,
+ defaultKeyValueStoreId: env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] || null,
+ internalPort: parseInt(env[ENV_VARS.INTERNAL_PORT], 10) || null,
};
-
- const promise = newPromise()
- .then(() => {
- if (!context.defaultKeyValueStoreId) return null;
- return getInput();
- })
- .then((input) => {
- context.input = input || null;
- return context;
- });
-
- return nodeifyPromise(promise, callback);
};
@@ -301,36 +213,14 @@ export const main = (userFunc) => {
try {
newPromise()
.then(() => {
- return getContext();
- })
- .catch((err) => {
- exitWithError(err, EXIT_CODES.ERROR_GETTING_INPUT, 'Failed to fetch act input:');
- })
- .then((context) => {
- if (!exited) {
- return userFunc(context);
- }
+ const context = getContext();
+ return userFunc(context);
})
.catch((err) => {
if (!exited) {
exitWithError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'User function threw an exception:');
}
})
- .then((userReturnValue) => {
- // Save output to the key-value store
- if (!exited && userReturnValue) {
- const output = {
- body: JSON.stringify(userReturnValue),
- contentType: 'application/json; charset=utf-8',
- };
- return setOutput(output);
- }
- })
- .catch((err) => {
- if (!exited) {
- exitWithError(err, EXIT_CODES.ERROR_SETTING_OUTPUT, 'Failed to save act output:');
- }
- })
.then(() => {
if (!exited) {
process.exit(EXIT_CODES.SUCCESS);
@@ -342,6 +232,8 @@ export const main = (userFunc) => {
}
};
+// TODO: this should rather be called Apify.listeningOnPort() or something like that
+
/**
* Notifies Apify runtime that act is listening on port specified by the APIFY_INTERNAL_PORT environment
* variable and is ready to receive a HTTP request with act input.
diff --git a/src/constants.js b/src/constants.js
index a19bc9b97..f00a85d3a 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -7,9 +7,7 @@
export const EXIT_CODES = {
SUCCESS: 0,
ERROR_USER_FUNCTION_THREW: 91,
- ERROR_GETTING_INPUT: 92,
- ERROR_SETTING_OUTPUT: 93,
- ERROR_UNKNOWN: 94,
+ ERROR_UNKNOWN: 92,
};
/**
diff --git a/src/index.js b/src/index.js
index 68bb4012c..c73074577 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,7 +1,7 @@
import EventEmitter from 'events';
import { setPromisesDependency, getPromisesDependency } from './utils';
-import { main, readyFreddy, getContext, getInput, setOutput, getValue, setValue, apifyClient } from './actor';
+import { main, readyFreddy, getContext, getValue, setValue, apifyClient } from './actor';
import { browse } from './browser';
/* globals module */
@@ -10,8 +10,6 @@ import { browse } from './browser';
const Apify = {
main,
getContext,
- getInput,
- setOutput,
getValue,
setValue,
readyFreddy,
diff --git a/test/actor.js b/test/actor.js
index 073612225..c7841c8ea 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -128,7 +128,7 @@ const testMain = (method, bodyRaw, contentType, userFunc, expectedExitCode = 0)
* Helper function that enables testing of Apify.main()
* @return Promise
*/
-const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputException, expectedOutput }) => {
+const testMain = ({ userFunc, context, exitCode }) => {
// Mock process.exit() to check exit code and prevent process exit
const processMock = sinon.mock(process);
const exitExpectation = processMock
@@ -137,51 +137,6 @@ const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputE
.once()
.returns();
- // Mock Apify.client.keyValueStores.getRecord() to test act input
- const kvStoresMock = sinon.mock(Apify.client.keyValueStores);
- if (mockInputException) {
- kvStoresMock
- .expects('getRecord')
- .throws(mockInputException);
- } else if (context.defaultKeyValueStoreId) {
- kvStoresMock
- .expects('getRecord')
- .withExactArgs({
- storeId: context.defaultKeyValueStoreId,
- promise: Apify.getPromisesDependency(),
- key: 'INPUT',
- })
- .returns(Promise.resolve(context.input))
- .once();
- } else {
- kvStoresMock
- .expects('putRecord')
- .never();
- }
-
- // Mock Apify.client.keyValueStores.putRecord() to test act output
- if (mockOutputException) {
- kvStoresMock
- .expects('putRecord')
- .throws(mockOutputException);
- } else if (context.defaultKeyValueStoreId && expectedOutput) {
- kvStoresMock
- .expects('putRecord')
- .withExactArgs({
- storeId: context.defaultKeyValueStoreId,
- promise: Apify.getPromisesDependency(),
- key: 'OUTPUT',
- contentType: expectedOutput.contentType,
- body: expectedOutput.body,
- })
- .returns(Promise.resolve())
- .once();
- } else {
- kvStoresMock
- .expects('putRecord')
- .never();
- }
-
// Mock APIFY_ environment variables
_.defaults(context, getEmptyContext());
setContextToEnv(context);
@@ -192,24 +147,18 @@ const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputE
.then(() => {
return new Promise((resolve, reject) => {
// Invoke main() function, the promise resolves after the user function is run
- // Note that if mockInputException is set, then user function will never get called!
- if (!mockInputException) {
- Apify.main((realContext) => {
- try {
- expect(realContext).to.eql(context);
- // Wait for all tasks in Node.js event loop to finish
- resolve();
- } catch (err) {
- reject(err);
- return;
- }
- // Call user func to test other behavior (note that it can throw)
- if (userFunc) return userFunc(realContext);
- });
- } else {
- Apify.main(() => {});
- resolve();
- }
+ Apify.main((realContext) => {
+ try {
+ expect(realContext).to.eql(context);
+ // Wait for all tasks in Node.js event loop to finish
+ resolve();
+ } catch (err) {
+ reject(err);
+ return;
+ }
+ // Call user func to test other behavior (note that it can throw)
+ if (userFunc) return userFunc(realContext);
+ });
});
})
.catch((err) => {
@@ -232,11 +181,9 @@ const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputE
.then(() => {
if (error) throw error;
processMock.verify();
- kvStoresMock.verify();
})
.finally(() => {
processMock.restore();
- kvStoresMock.restore();
});
};
@@ -251,7 +198,6 @@ const getEmptyContext = () => {
startedAt: null,
timeoutAt: null,
defaultKeyValueStoreId: null,
- input: null,
};
};
@@ -280,13 +226,11 @@ describe('Apify.getContext()', () => {
const expectedContext = getEmptyContext();
setContextToEnv(expectedContext);
- return Apify.getContext()
- .then((context) => {
- expect(context).to.eql(expectedContext);
- });
+ const context = Apify.getContext();
+ expect(context).to.eql(expectedContext);
});
- it('works with with non-null values / no input', () => {
+ it('works with with non-null values', () => {
const expectedContext = _.extend(getEmptyContext(), {
internalPort: 12345,
actId: 'test actId',
@@ -295,191 +239,15 @@ describe('Apify.getContext()', () => {
token: 'auth token',
startedAt: new Date('2017-01-01'),
timeoutAt: new Date(),
- defaultKeyValueStoreId: null,
- input: null,
+ defaultKeyValueStoreId: 'some store',
});
setContextToEnv(expectedContext);
- return Apify.getContext()
- .then((context) => {
- expect(context).to.eql(expectedContext);
- });
- });
-
- it('works with callbacks', () => {
- const expectedContext = _.extend(getEmptyContext(), {
- internalPort: 4455,
- actId: 'test actId x',
- actRunId: 'test actId x',
- startedAt: new Date('2017-01-01'),
- timeoutAt: new Date(),
- });
- setContextToEnv(expectedContext);
-
- return new Promise((resolve, reject) => {
- Apify.getContext((err, context) => {
- if (err) reject(err);
- resolve(context);
- });
- })
- .then((context) => {
- expect(context).to.eql(expectedContext);
- });
- });
-
- it('works with with non-null values / text input', () => {
- const expectedContext = {
- internalPort: 12345,
- actId: 'test actId',
- actRunId: 'test actId',
- userId: 'some user',
- token: 'auth token',
- startedAt: new Date('2017-01-01'),
- timeoutAt: new Date(),
- defaultKeyValueStoreId: 'test storeId',
- input: {
- body: 'test body',
- contentType: 'text/plain',
- },
- };
- setContextToEnv(expectedContext);
-
- const mock = sinon.mock(Apify.client.keyValueStores);
- const expectation = mock.expects('getRecord');
- expectation
- .withExactArgs({
- storeId: expectedContext.defaultKeyValueStoreId,
- promise: Apify.getPromisesDependency(),
- key: 'INPUT',
- })
- .once()
- .returns(Promise.resolve(expectedContext.input));
-
- return Promise.resolve()
- .then(() => {
- return Apify.getContext();
- })
- .then((context) => {
- expect(context).to.eql(expectedContext);
- expectation.verify();
- })
- .finally(() => {
- mock.restore();
- });
+ const context = Apify.getContext();
+ expect(context).to.eql(expectedContext);
});
});
-describe('Apify.getInput()', () => {
- it('supports both promises and callbacks (on success)', () => {
- const mock = sinon.mock(Apify.client.keyValueStores);
- mock.expects('getRecord')
- .twice()
- .returns(Promise.resolve(null));
-
- return Promise.resolve()
- .then(() => {
- // test promise
- return Apify.getInput();
- })
- .then((input) => {
- expect(input).to.be.eql(null);
- })
- .then(() => {
- // test callback
- return new Promise((resolve, reject) => {
- return Apify.getInput((err, input) => {
- if (err) return reject(err);
- resolve(input);
- });
- });
- })
- .then((input) => {
- expect(input).to.be.eql(null);
-
- mock.verify();
- })
- .finally(() => {
- mock.restore();
- });
- });
-
- it('supports both promises and callbacks (on error)', () => {
- const mock = sinon.mock(Apify.client.keyValueStores);
- mock.expects('getRecord')
- .twice()
- .throws(new Error('Test error'));
-
- return Promise.resolve()
- .then(() => {
- // test promise
- return Apify.getInput();
- })
- .catch((err) => {
- expect(err.message).to.be.eql('Test error');
- })
- .then(() => {
- // test callback
- return new Promise((resolve, reject) => {
- return Apify.getInput((err, input) => {
- if (err) return reject(err);
- resolve(input);
- });
- });
- })
- .catch((err) => {
- expect(err.message).to.be.eql('Test error');
- })
- .then(() => {
- mock.verify();
- })
- .finally(() => {
- mock.restore();
- });
- });
-
- it('returns null on undefined keyValueStores.getRecord() result', () => {
- const mock = sinon.mock(Apify.client.keyValueStores);
- mock.expects('getRecord')
- .once()
- .returns(Promise.resolve(undefined));
-
- return Promise.resolve()
- .then(() => {
- return Apify.getInput();
- })
- .then((input) => {
- expect(input).to.be.eql(null);
- mock.verify();
- })
- .finally(() => {
- mock.restore();
- });
- });
-
- it('fails on invalid keyValueStores.getRecord() result', () => {
- const mock = sinon.mock(Apify.client.keyValueStores);
- mock.expects('getRecord')
- .once()
- .returns(Promise.resolve({ invalid: 'bla bla' }));
-
- return Promise.resolve()
- .then(() => {
- return Apify.getInput();
- })
- .then(() => {
- expect.fail();
- })
- .catch((err) => {
- expect(err.message).to.contain('ApifyClient returned an unexpected value');
- })
- .then(() => {
- mock.verify();
- })
- .finally(() => {
- mock.restore();
- });
- });
-});
describe('Apify.main()', () => {
it('throws on invalid args', () => {
@@ -516,60 +284,7 @@ describe('Apify.main()', () => {
});
});
- it('gets input correctly', () => {
- const context = {
- defaultKeyValueStoreId: 'test storeId',
- input: {
- body: 'test body',
- contentType: 'text/plain',
- },
- };
- return testMain({
- userFunc: null,
- context,
- exitCode: 0,
- });
- });
-
- it('sets output from simple user function', () => {
- const context = {
- defaultKeyValueStoreId: 'test storeId x',
- };
- const output = { test: 123 };
- return testMain({
- userFunc: () => {
- return output;
- },
- context,
- exitCode: 0,
- expectedOutput: {
- contentType: 'application/json; charset=utf-8',
- body: JSON.stringify(output),
- },
- });
- });
-
- it('sets output from promised user function', () => {
- const context = {
- defaultKeyValueStoreId: 'test storeId x',
- };
- const output = { test: 123 };
- return testMain({
- userFunc: () => {
- return Promise.resolve().then(() => {
- return output;
- });
- },
- context,
- exitCode: 0,
- expectedOutput: {
- contentType: 'application/json; charset=utf-8',
- body: JSON.stringify(output),
- },
- });
- });
-
- it('on exception in simple user function the process exits with code 1001', () => {
+ it('on exception in simple user function the process exits with code 91', () => {
return testMain({
userFunc: () => {
throw new Error('Test exception I');
@@ -579,7 +294,7 @@ describe('Apify.main()', () => {
});
});
- it('on exception in promised user function the process exits with code 1001', () => {
+ it('on exception in promised user function the process exits with code 91', () => {
return testMain({
userFunc: () => {
return new Promise((resolve) => {
@@ -593,82 +308,12 @@ describe('Apify.main()', () => {
exitCode: 91,
});
});
-
- it('on exception in getInput the process exits with code 1002', () => {
- return testMain({
- userFunc: null,
- context: {
- defaultKeyValueStoreId: 'test storeId',
- input: {},
- },
- exitCode: 92,
- mockInputException: new Error('Text exception III'),
- });
- });
-
- it('on exception in setOutput the process exits with code 1003', () => {
- return testMain({
- userFunc: () => {
- return 'anything';
- },
- context: {
- defaultKeyValueStoreId: 'test storeId',
- },
- exitCode: 93,
- mockOutputException: new Error('Text exception IV'),
- });
- });
-
- it('on non-JSON-stringifyable return value the process exits with code 1003', () => {
- const circularObj = {};
- circularObj.aaa = circularObj;
-
- return testMain({
- userFunc: () => {
- return circularObj;
- },
- context: {
- defaultKeyValueStoreId: 'test storeId',
- },
- exitCode: 93,
- });
- });
});
-describe('Apify.setOutput()', () => {
- it('throws on invalid args', () => {
- expect(() => {
- Apify.setOutput();
- }).to.throw(Error);
-
- expect(() => {
- Apify.setOutput('bla bla');
- }).to.throw(Error);
-
- expect(() => {
- Apify.setOutput(1234);
- }).to.throw(Error);
-
- expect(() => {
- Apify.setOutput({});
- }).to.throw(Error);
-
- expect(() => {
- Apify.setOutput({ body: undefined, contentType: 'test' });
- }).to.throw(Error);
-
- expect(() => {
- Apify.setOutput({ body: {}, contentType: 456 });
- }).to.throw(Error);
-
- expect(() => {
- Apify.setOutput({ body: {}, contentType: undefined });
- }).to.throw(Error);
- });
-});
describe('Apify.getValue()', () => {
it('throws on invalid args', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const keyErrMsg = 'The "key" parameter must be a non-empty string';
expect(() => { Apify.getValue(); }).to.throw(Error, keyErrMsg);
expect(() => { Apify.getValue({}); }).to.throw(Error, keyErrMsg);
@@ -676,7 +321,18 @@ describe('Apify.getValue()', () => {
expect(() => { Apify.getValue(null); }).to.throw(Error, keyErrMsg);
});
+ it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', () => {
+ const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';
+
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '';
+ expect(() => { Apify.getValue('KEY'); }).to.throw(Error, errMsg);
+
+ delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
+ expect(() => { Apify.getValue('some other key'); }).to.throw(Error, errMsg);
+ });
+
it('supports both promises and callbacks (on success)', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const mock = sinon.mock(Apify.client.keyValueStores);
mock.expects('getRecord')
.twice()
@@ -710,6 +366,7 @@ describe('Apify.getValue()', () => {
});
it('supports both promises and callbacks (on error)', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const mock = sinon.mock(Apify.client.keyValueStores);
mock.expects('getRecord')
.twice()
@@ -744,6 +401,7 @@ describe('Apify.getValue()', () => {
});
it('returns null on undefined keyValueStores.getRecord() result', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const mock = sinon.mock(Apify.client.keyValueStores);
mock.expects('getRecord')
.once()
@@ -763,6 +421,7 @@ describe('Apify.getValue()', () => {
});
it('fails on invalid keyValueStores.getRecord() result', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const mock = sinon.mock(Apify.client.keyValueStores);
mock.expects('getRecord')
.once()
@@ -789,6 +448,7 @@ describe('Apify.getValue()', () => {
describe('Apify.setValue()', () => {
it('throws on invalid args', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const keyErrMsg = 'The "key" parameter must be a non-empty string';
expect(() => { Apify.setValue(); }).to.throw(Error, keyErrMsg);
expect(() => { Apify.setValue('', null); }).to.throw(Error, keyErrMsg);
@@ -821,11 +481,22 @@ describe('Apify.setValue()', () => {
expect(() => { Apify.setValue('key', 'value', ''); }).to.throw(Error, contTypeStringErrMsg);
});
+ it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', () => {
+ const errMsg = 'The \'APIFY_DEFAULT_KEY_VALUE_STORE_ID\' environment variable is not defined';
+
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '';
+ expect(() => { Apify.setValue('KEY', { something: 123 }); }).to.throw(Error, errMsg);
+
+ delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;
+ expect(() => { Apify.setValue('KEY', { something: 123 }); }).to.throw(Error, errMsg);
+ });
+
it('supports both promises and callbacks (on success)', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const mock = sinon.mock(Apify.client.keyValueStores);
mock.expects('putRecord')
- .exactly(4)
- .returns(Promise.resolve(null));
+ .exactly(4)
+ .returns(Promise.resolve(null));
return Promise.resolve()
.then(() => {
@@ -863,6 +534,7 @@ describe('Apify.setValue()', () => {
});
it('supports both promises and callbacks (on error)', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const mock = sinon.mock(Apify.client.keyValueStores);
mock.expects('putRecord')
.twice()
@@ -897,6 +569,7 @@ describe('Apify.setValue()', () => {
});
it('correctly stores object values as JSON', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const storeId = 'mystore';
const key = 'mykey';
const value = { someValue: 123 };
@@ -931,6 +604,7 @@ describe('Apify.setValue()', () => {
});
it('correctly stores raw string values', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const storeId = 'mystore3';
const key = 'mykey2';
const value = 'some string value';
@@ -966,6 +640,7 @@ describe('Apify.setValue()', () => {
});
it('correctly stores raw Buffer values', () => {
+ process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const storeId = 'mystore3';
const key = 'mykey2';
const value = Buffer.from('some text value');