Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

More robust Apifier.getInput() and setOutput(), new unit tests, updated README.md, bumped package.json version

Jan Curnd13be9683f19
diff --git a/README.md b/README.md
index a86b1fbe6..b6d5afb28 100644
--- a/README.md
+++ b/README.md
@@ -182,6 +182,18 @@ If `Apifier.setPromisesDependency()` is not called, the runtime defaults to
native promises if they are available, or it throws an error.
+### Miscellaneous
+
+`Apifier.client` property contains a reference to `ApifyClient` instance from the
+[apify-client](https://www.npmjs.com/package/apify-client) NPM package
+that is used for all underlying calls to Apifier API.
+You can use this to set options to the client.
+
+`Apifier.events` property contains a reference to an `EventEmitter` instance
+that is used by Actor runtime to notify your process about various events.
+This will be used in the future.
+
+
### Internal web server
**TODO: this is still not finished**
diff --git a/package.json b/package.json
index d12ce52be..6ceae1ae7 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apifier",
- "version": "0.0.31",
+ "version": "0.0.32",
"description": "Apifier runtime for JavaScript",
"main": "build/index.js",
"keywords": [
@@ -47,6 +47,7 @@
"babel-eslint": "^7.2.2",
"babel-preset-es2015": "^6.1.18",
"babel-preset-stage-0": "^6.1.18",
+ "bluebird": "^3.5.0",
"chai": "^3.4.1",
"eslint": "^3.19.0",
"eslint-config-airbnb": "^14.1.0",
diff --git a/src/actor.js b/src/actor.js
index f87778cb5..07404c65d 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -2,9 +2,9 @@ import fs from 'fs';
import url from 'url';
import ApifyClient from 'apify-client';
import { APIFY_ENV_VARS, EXIT_CODES, KV_STORE_KEYS } from './constants';
-import { getPromisesDependency, newPromise, nodeifyPromise } from './utils';
+import { getPromisePrototype, newPromise, nodeifyPromise } from './utils';
-/* global process */
+/* global process, Buffer */
// TODO: protocol/host/port/basePath should be replaced with baseUrl
const clientOpts = {};
@@ -44,17 +44,33 @@ const getDefaultStoreIdOrThrow = () => {
* This data is stored in the key-value store created specifically for this run,
* whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable,
* under record with the `INPUT` key.
- * The result of the function is an object such as `{ body: String/Buffer, contentType: String }`,
+ * The result of the function is an object such as `{ body: String|Buffer, contentType: String|null }`,
* or `null` if record was not found.
* @param callback Optional callback.
* @return Returns a promise if no callback was provided, otherwise the return value is not defined.
*/
export const getInput = (callback = null) => {
- return apifyClient.keyValueStores.getRecord({
- storeId: getDefaultStoreIdOrThrow(),
- promise: getPromisesDependency(),
- recordKey: KV_STORE_KEYS.INPUT,
- }, callback);
+ const promise = newPromise()
+ .then(() => {
+ return apifyClient.keyValueStores.getRecord({
+ storeId: getDefaultStoreIdOrThrow(),
+ promise: getPromisePrototype(),
+ recordKey: KV_STORE_KEYS.INPUT,
+ });
+ })
+ .then((input) => {
+ // Ensure we always return null or { body: String|Buffer, contentType: String|null } to user
+ if (!input) {
+ input = null;
+ } else if (typeof (input) !== 'object'
+ || (typeof (input.body) !== 'string' && !Buffer.isBuffer(input.body))
+ || (typeof (input.contentType) !== 'string' && input.contentType !== null)) {
+ throw new Error('ApifyClient returned an unexpected value from keyValueStores.getRecord()');
+ }
+ return input;
+ });
+
+ return nodeifyPromise(promise, callback);
};
/**
@@ -62,21 +78,34 @@ export const getInput = (callback = null) => {
* This data is stored in the key-value store created specifically for this run,
* whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable,
* under record with the `OUTPUT` key.
- * The function has no result.
- * @param output
+ * The function has no result, but throws on invalid args.
+ * @param output Must be an object such as { body: String|Buffer, contentType: String|null }
* @param callback Optional callback.
* @return Returns a promise if no callback was provided, otherwise the return value is not defined.
*/
export const setOutput = (output, callback = null) => {
- if (!output) throw new Error('The "output" parameter must be provided.');
-
- return apifyClient.keyValueStores.putRecord({
- storeId: getDefaultStoreIdOrThrow(),
- promise: getPromisesDependency(),
- recordKey: KV_STORE_KEYS.OUTPUT,
- body: output.body,
- contentType: output.contentType,
- }, callback);
+ if (typeof (output) !== 'object') {
+ throw new Error('The "output" parameter must be an object.');
+ }
+ if (typeof (output.body) !== 'string' && !Buffer.isBuffer(output.body)) {
+ throw new Error('The "output.body" parameter must be String or Buffer.');
+ }
+ if (typeof (output.contentType) !== 'string' && output.contentType !== null) {
+ throw new Error('The "output.contentType" parameter must be String or null.');
+ }
+
+ const promise = newPromise()
+ .then(() => {
+ return apifyClient.keyValueStores.putRecord({
+ storeId: getDefaultStoreIdOrThrow(),
+ promise: getPromisePrototype(),
+ recordKey: KV_STORE_KEYS.OUTPUT,
+ body: output.body,
+ contentType: output.contentType,
+ });
+ });
+
+ return nodeifyPromise(promise, callback);
};
/**
@@ -121,6 +150,7 @@ export const getContext = (callback = null) => {
context.input = input || null;
return context;
});
+
return nodeifyPromise(promise, callback);
};
diff --git a/src/utils.js b/src/utils.js
index 22881a2f6..8814e004f 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -21,25 +21,38 @@ export const getPromisesDependency = () => {
};
/**
- * Returns a result of `Promise.resolve()` using promise library set by `setPromisesDependency()`,
- * or using native promises, or throws if no native promises are available.
- * @return {*}
+ * Gets a promise dependency set using `setPromisesDependency()`,
+ * or returns the native `Promise` function, or throws if no native promises are available.
+ * @return Promise
*/
-export const newPromise = () => {
+export const getPromisePrototype = () => {
if (PromisesDependency) {
if (typeof (PromisesDependency.resolve) !== 'function') {
throw new Error('The promise dependency set using Apifier.setPromisesDependency() does not define resolve() function.');
}
- return PromisesDependency.resolve();
+ return PromisesDependency;
}
- if (typeof Promise === 'function') return Promise.resolve();
+ if (typeof Promise === 'function') return Promise;
throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');
};
+/**
+ * Returns a result of `Promise.resolve()` using promise library set by `setPromisesDependency()`,
+ * or using native promises, or throws if no native promises are available.
+ * @return {*}
+ */
+export const newPromise = () => {
+ return getPromisePrototype().resolve();
+};
+
export const nodeifyPromise = (promise, callback) => {
if (!promise) throw new Error('The "promise" parameter must be provided.');
- if (!callback) return promise;
- promise
- .then(() => callback(context))
- .catch(err => callback(err));
+
+ if (callback) {
+ promise
+ .then(result => callback(null, result))
+ .catch(err => callback(err));
+ } else {
+ return promise;
+ }
};
diff --git a/test/actor.js b/test/actor.js
index 7cca5118b..f880969d3 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -3,17 +3,18 @@ import _ from 'underscore';
import { expect } from 'chai';
import sinon from 'sinon';
import tmp from 'tmp';
+import Promise from 'bluebird';
/* global process */
// TODO: override console.log() to test the error messages (now they are printed to console)
+// TODO: test callback version of functions!!!
// NOTE: use require() here because this is how its done in acts
const Apifier = process.env.TEST_BABEL_BUILD ? require('../build/index') : require('../src/index');
if (process.env.TEST_BABEL_BUILD) console.log('Running with TEST_BABEL_BUILD option');
-process.test = '1243';
/*
let freePorts = [];
@@ -30,13 +31,6 @@ before(() => {
const popFreePort = () => freePorts.pop();
*/
-// always restore original process.exit()
-const origProcessExit = process.exit;
-after(() => {
- process.exit = origProcessExit;
-});
-
-
const createWatchFile = () => {
const tmpobj = tmp.fileSync();
const path = tmpobj.name;
@@ -135,7 +129,7 @@ const testMain = (method, bodyRaw, contentType, userFunc, expectedExitCode = 0)
/**
* Helper function that enables testing of Apifier.main()
- * @return A promise
+ * @return Promise
*/
const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputException, expectedOutput }) => {
// Mock process.exit() to check exit code and prevent process exit
@@ -159,7 +153,7 @@ const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputE
storeId: context.defaultKeyValueStoreId,
promise: Apifier.getPromisesDependency(),
recordKey: 'INPUT',
- }, null)
+ })
.returns(Promise.resolve(context.input))
.once();
} else {
@@ -182,7 +176,7 @@ const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputE
recordKey: 'OUTPUT',
contentType: expectedOutput.contentType,
body: expectedOutput.body,
- }, null)
+ })
.returns(Promise.resolve())
.once();
} else {
@@ -197,55 +191,56 @@ const testMain = ({ userFunc, context, exitCode, mockInputException, mockOutputE
let error = null;
- return new Promise((resolve, reject) => {
- // Invoke main() function, the promise resolves after the user function is run
- // Note that if mockInputException is set, then user function will never get called!
- if (!mockInputException) {
- Apifier.main((realContext) => {
- try {
- expect(realContext).to.eql(context);
- // Wait for all tasks in Node.js event loop to finish
+ return Promise.resolve()
+ .then(() => {
+ return new Promise((resolve, reject) => {
+ // Invoke main() function, the promise resolves after the user function is run
+ // Note that if mockInputException is set, then user function will never get called!
+ if (!mockInputException) {
+ Apifier.main((realContext) => {
+ try {
+ expect(realContext).to.eql(context);
+ // Wait for all tasks in Node.js event loop to finish
+ resolve();
+ } catch (err) {
+ reject(err);
+ return;
+ }
+ // Call user func to test other behavior (note that it can throw)
+ if (userFunc) return userFunc(realContext);
+ });
+ } else {
+ Apifier.main(() => {});
resolve();
- } catch (err) {
- reject(err);
- return;
}
- // Call user func to test other behavior (note that it can throw)
- if (userFunc) return userFunc(realContext);
});
- } else {
- Apifier.main(() => {});
- resolve();
- }
- })
- .catch((err) => {
- error = err;
- })
- .then(() => {
- // Waits max 1000 millis for process.exit() mock to be called
- // console.log(`XXX: grand finale: ${err}`);
- return new Promise((resolve) => {
- const waitUntil = Date.now() + 1000;
- const intervalId = setInterval(() => {
- // console.log('test for exitExpectation.called');
- if (!exitExpectation.called && Date.now() < waitUntil) return;
- clearInterval(intervalId);
- // console.log(`exitExpectation.called: ${exitExpectation.called}`);
- resolve();
- }, 10);
+ })
+ .catch((err) => {
+ error = err;
+ })
+ .then(() => {
+ // Waits max 1000 millis for process.exit() mock to be called
+ // console.log(`XXX: grand finale: ${err}`);
+ return new Promise((resolve) => {
+ const waitUntil = Date.now() + 1000;
+ const intervalId = setInterval(() => {
+ // console.log('test for exitExpectation.called');
+ if (!exitExpectation.called && Date.now() < waitUntil) return;
+ clearInterval(intervalId);
+ // console.log(`exitExpectation.called: ${exitExpectation.called}`);
+ resolve();
+ }, 10);
+ });
+ })
+ .then(() => {
+ if (error) throw error;
+ processMock.verify();
+ kvStoresMock.verify();
+ })
+ .finally(() => {
+ processMock.restore();
+ kvStoresMock.restore();
});
- })
- .then(() => {
- // Restore mocked functions and verify they were called correctly
- // console.log('XXX: restore');
- processMock.restore();
- kvStoresMock.restore();
-
- if (error) throw error;
-
- processMock.verify();
- kvStoresMock.verify();
- });
};
@@ -306,6 +301,27 @@ describe('Apifier.getContext()', () => {
});
});
+ it('works with callbacks', () => {
+ const expectedContext = _.extend(getEmptyContext(), {
+ internalPort: 4455,
+ actId: 'test actId x',
+ actRunId: 'test actId x',
+ startedAt: new Date('2017-01-01'),
+ timeoutAt: new Date(),
+ });
+ setContextToEnv(expectedContext);
+
+ return new Promise((resolve, reject) => {
+ Apifier.getContext((err, context) => {
+ if (err) reject(err);
+ resolve(context);
+ });
+ })
+ .then((context) => {
+ expect(context).to.eql(expectedContext);
+ });
+ });
+
it('works with with non-null values / text input', () => {
const expectedContext = {
internalPort: 12345,
@@ -328,19 +344,132 @@ describe('Apifier.getContext()', () => {
storeId: expectedContext.defaultKeyValueStoreId,
promise: Apifier.getPromisesDependency(),
recordKey: 'INPUT',
- }, null)
+ })
.once()
.returns(Promise.resolve(expectedContext.input));
- return Apifier.getContext()
+ return Promise.resolve()
+ .then(() => {
+ return Apifier.getContext();
+ })
.then((context) => {
expect(context).to.eql(expectedContext);
- mock.restore();
expectation.verify();
})
+ .finally(() => {
+ mock.restore();
+ });
+ });
+});
+
+describe('Apifier.getInput()', () => {
+ it('supports both promises and callbacks (on success)', () => {
+ const mock = sinon.mock(Apifier.client.keyValueStores);
+ mock.expects('getRecord')
+ .twice()
+ .returns(Promise.resolve(null));
+
+ return Promise.resolve()
+ .then(() => {
+ // test promise
+ return Apifier.getInput();
+ })
+ .then((input) => {
+ expect(input).to.be.eql(null);
+ })
+ .then(() => {
+ // test callback
+ return new Promise((resolve, reject) => {
+ return Apifier.getInput((err, input) => {
+ if (err) return reject(err);
+ resolve(input);
+ });
+ });
+ })
+ .then((input) => {
+ expect(input).to.be.eql(null);
+
+ mock.verify();
+ })
+ .finally(() => {
+ mock.restore();
+ });
+ });
+
+ it('supports both promises and callbacks (on error)', () => {
+ const mock = sinon.mock(Apifier.client.keyValueStores);
+ mock.expects('getRecord')
+ .twice()
+ .throws(new Error('Test error'));
+
+ return Promise.resolve()
+ .then(() => {
+ // test promise
+ return Apifier.getInput();
+ })
.catch((err) => {
+ expect(err.message).to.be.eql('Test error');
+ })
+ .then(() => {
+ // test callback
+ return new Promise((resolve, reject) => {
+ return Apifier.getInput((err, input) => {
+ if (err) return reject(err);
+ resolve(input);
+ });
+ });
+ })
+ .catch((err) => {
+ expect(err.message).to.be.eql('Test error');
+ })
+ .then(() => {
+ mock.verify();
+ })
+ .finally(() => {
+ mock.restore();
+ });
+ });
+
+ it('returns null on undefined keyValueStores.getRecord() result', () => {
+ const mock = sinon.mock(Apifier.client.keyValueStores);
+ mock.expects('getRecord')
+ .once()
+ .returns(Promise.resolve(undefined));
+
+ return Promise.resolve()
+ .then(() => {
+ return Apifier.getInput();
+ })
+ .then((input) => {
+ expect(input).to.be.eql(null);
+ mock.verify();
+ })
+ .finally(() => {
+ mock.restore();
+ });
+ });
+
+ it('fails on invalid keyValueStores.getRecord() result', () => {
+ const mock = sinon.mock(Apifier.client.keyValueStores);
+ mock.expects('getRecord')
+ .once()
+ .returns(Promise.resolve({ invalid: 'bla bla' }));
+
+ return Promise.resolve()
+ .then(() => {
+ return Apifier.getInput();
+ })
+ .then(() => {
+ expect.fail();
+ })
+ .catch((err) => {
+ expect(err.message).to.contain('ApifyClient returned an unexpected value');
+ })
+ .then(() => {
+ mock.verify();
+ })
+ .finally(() => {
mock.restore();
- throw err;
});
});
});
@@ -500,6 +629,39 @@ describe('Apifier.main()', () => {
});
+describe('Apifier.setOuput()', () => {
+ it('throws on invalid args', () => {
+ expect(() => {
+ Apifier.setOutput();
+ }).to.throw(Error);
+
+ expect(() => {
+ Apifier.setOutput('bla bla');
+ }).to.throw(Error);
+
+ expect(() => {
+ Apifier.setOutput(1234);
+ }).to.throw(Error);
+
+ expect(() => {
+ Apifier.setOutput({});
+ }).to.throw(Error);
+
+ expect(() => {
+ Apifier.setOutput({ body: undefined, contentType: 'test' });
+ }).to.throw(Error);
+
+ expect(() => {
+ Apifier.setOutput({ body: {}, contentType: 456 });
+ }).to.throw(Error);
+
+ expect(() => {
+ Apifier.setOutput({ body: {}, contentType: undefined });
+ }).to.throw(Error);
+ });
+});
+
+
describe('Apifier.events', () => {
it('is there and works as EventEmitter', () => {
return new Promise((resolve, reject) => {
diff --git a/test/utils.js b/test/utils.js
index 78fd79b57..4f1032853 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -1,3 +1,4 @@
+import BluebirdPromise from 'bluebird';
import { expect } from 'chai';
import * as utils from '../src/utils';
@@ -24,12 +25,14 @@ describe('Apifier.xxxPromisesDependency()', () => {
expect(Apifier.getPromisesDependency()).to.be.a('null');
expect(utils.newPromise()).to.have.property('then');
+ // Check native promise
Apifier.setPromisesDependency(Promise);
expect(Apifier.getPromisesDependency()).to.equal(Promise);
expect(utils.newPromise()).to.have.property('then');
+
+ // Check bluebird
+ Apifier.setPromisesDependency(BluebirdPromise);
+ expect(Apifier.getPromisesDependency()).to.equal(BluebirdPromise);
+ expect(utils.newPromise()).to.have.property('then');
});
});
-
-
-// TODO: test nodeifyPromise() !!!
-