Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added unit tests for Apifier.main() output

Jan Curn99e92cd1be6a
diff --git a/src/actor.js b/src/actor.js
index d7532b98f..94c4dd2ab 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -58,7 +58,7 @@ export const getInput = (callback = null) => {
export const setOutput = (output, callback = null) => {
if (!output) throw new Error('The "output" parameter must be provided.');
- return apifyClient.keyValueStores.setRecord({
+ return apifyClient.keyValueStores.putRecord({
storeId: getDefaultStoreIdOrThrow(),
promise: getPromisesDependency(),
body: output.body,
@@ -151,7 +151,7 @@ export const main = (userFunc) => {
if (!exited && userReturnValue) {
const output = {
body: JSON.stringify(userReturnValue),
- contentType: 'application/json',
+ contentType: 'application/json; charset=utf-8',
};
return setOutput(output);
}
diff --git a/test/actor.js b/test/actor.js
index 50e346698..16e820c2b 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -137,7 +137,7 @@ const testMain = (method, bodyRaw, contentType, userFunc, expectedExitCode = 0)
* Helper function that enables testing of Apifier.main()
* @return A promise
*/
-const testMain = ({ userFunc, context, exitCode, mockInputException }) => {
+const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputException, expectedOutput }) => {
// Mock process.exit() to check exit code and prevent process exit
const processMock = sinon.mock(process);
const exitExpectation = processMock
@@ -146,7 +146,7 @@ const testMain = ({ userFunc, context, exitCode, mockInputException }) => {
.once()
.returns();
- // Mock Apifier.client.keyValueStores.getRecord() for input getter
+ // Mock Apifier.client.keyValueStores.getRecord() to test act input
const kvStoresMock = sinon.mock(Apifier.client.keyValueStores);
if (mockInputException) {
kvStoresMock
@@ -161,14 +161,38 @@ const testMain = ({ userFunc, context, exitCode, mockInputException }) => {
}, null)
.returns(Promise.resolve(context.input))
.once();
+ } else {
+ kvStoresMock
+ .expects('putRecord')
+ .never();
+ }
+
+ // Mock Apifier.client.keyValueStores.putRecord() to test act output
+ if (mockOutputException) {
+ kvStoresMock
+ .expects('putRecord')
+ .throws(mockOutputException);
+ } else if (context.defaultKeyValueStoreId && expectedOutput) {
+ kvStoresMock
+ .expects('putRecord')
+ .withExactArgs({
+ storeId: context.defaultKeyValueStoreId,
+ promise: Apifier.getPromisesDependency(),
+ contentType: expectedOutput.contentType,
+ body: expectedOutput.body,
+ }, null)
+ .returns(Promise.resolve())
+ .once();
+ } else {
+ kvStoresMock
+ .expects('putRecord')
+ .never();
}
// Mock APIFY_ environment variables
_.defaults(context, getEmptyContext());
setContextToEnv(context);
- // TODO: mock Apifier.client.keyValueStores.putRecord() for output setter
-
let error = null;
return new Promise((resolve, reject) => {
@@ -370,18 +394,40 @@ describe('Apifier.main()', () => {
it('sets output from simple user function', () => {
const context = {
- defaultKeyValueStoreId: 'test storeId',
- input: null,
+ defaultKeyValueStoreId: 'test storeId x',
};
+ const output = { test: 123 };
return testMain({
- userFunc: null,
+ userFunc: () => {
+ return output;
+ },
context,
exitCode: 0,
+ expectedOutput: {
+ contentType: 'application/json; charset=utf-8',
+ body: JSON.stringify(output),
+ },
});
});
it('sets output from promised user function', () => {
- // TODO
+ const context = {
+ defaultKeyValueStoreId: 'test storeId x',
+ };
+ const output = { test: 123 };
+ return testMain({
+ userFunc: () => {
+ return Promise.resolve().then(() => {
+ return output;
+ });
+ },
+ context,
+ exitCode: 0,
+ expectedOutput: {
+ contentType: 'application/json; charset=utf-8',
+ body: JSON.stringify(output),
+ },
+ });
});
it('on exception in simple user function the process exits with code 1001', () => {
@@ -420,6 +466,19 @@ describe('Apifier.main()', () => {
mockInputException: new Error('Text exception III'),
});
});
+
+ it('on exception in setInput the process exits with code 1003', () => {
+ return testMain({
+ userFunc: () => {
+ return 'anything';
+ },
+ context: {
+ defaultKeyValueStoreId: 'test storeId',
+ },
+ exitCode: 1003,
+ mockOutputException: new Error('Text exception IV'),
+ });
+ });
});