Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Removed old code and dependencies, fixed package.json & bumped version, added new unit tests
Jan Curn53db128d8a04
diff --git a/package.json b/package.jsonindex 8059b01e8..77d4b2c9e 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apifier",- "version": "0.0.22",+ "version": "0.0.23","description": "Apifier SDK for JavaScript","main": "build/index.js","keywords": [@@ -33,15 +33,13 @@"build": "babel src --out-dir build","test": "npm run build && mocha --compilers js:babel-core/register --recursive","test-cov": "npm run build && babel-node node_modules/isparta/bin/isparta cover --report html --report text node_modules/.bin/_mocha",- "prepare": "npm run build && TEST_BABEL_BUILD=1 npm test && npm run lint",+ "prepare": "npm run build",+ "prepublishOnly": "TEST_BABEL_BUILD=1 npm test && npm run lint","clean": "rm -rf build","lint": "npm run build && eslint src test"},"dependencies": {- "apify-client": "^0.0.1",- "request": "^2.81.0",- "request-promise": "^4.2.0",- "underscore": "^1.8.3"+ "apify-client": "^0.0.1"},"devDependencies": {"babel-cli": "^6.6.5",@@ -62,6 +60,7 @@"portastic": "^1.0.1","sinon": "^2.2.0","sinon-stub-promise": "^4.0.0",- "tmp": "^0.0.31"+ "tmp": "^0.0.31",+ "underscore": "^1.8.3"}}diff --git a/src/actor.js b/src/actor.jsindex d03fb3512..d7532b98f 100644--- a/src/actor.js+++ b/src/actor.js@@ -1,11 +1,14 @@import fs from 'fs';import ApifyClient from 'apify-client';import { APIFY_ENV_VARS, EXIT_CODES } from './constants';-import { getPromisesDependency, newPromise } from './utils';+import { getPromisesDependency, newPromise, nodeifyPromise } from './utils';/* global process */-const apifyClient = new ApifyClient();+/**+ * Exported to enable unit testing mock-up.+ */+export const apifyClient = new ApifyClient();/**@@ -80,14 +83,14 @@ export const setOutput = (output, callback = null) => {* }* ```* The information is generate using the APIFY_XXX environment variables and the input data is fetched from Apifier API.- * If some of the values cannot be determined, it will be set to null;- * we don't throw error in such a case in order to simplify local development and debugging of acts.+ * If some of the variables is not defined or is invalid, the corresponding value in the context object will be null;+ * an error is not thrown in such a case in order to simplify local development and debugging of acts.* @param callback Optional callback.* @return Returns a promise if no callback was provided, otherwise the return value is not defined.*/export const getContext = (callback = null) => {const context = {- internalPort: process.env[APIFY_ENV_VARS.INTERNAL_PORT] || null,+ internalPort: parseInt(process.env[APIFY_ENV_VARS.INTERNAL_PORT], 10) || null,actId: process.env[APIFY_ENV_VARS.ACT_ID] || null,actRunId: process.env[APIFY_ENV_VARS.ACT_RUN_ID] || null,startedAt: tryParseDate(process.env[APIFY_ENV_VARS.STARTED_AT]) || null,@@ -96,7 +99,7 @@ export const getContext = (callback = null) => {input: null,};- return newPromise()+ const promise = newPromise().then(() => {if (!context.defaultKeyValueStoreId) return null;return getInput();@@ -104,18 +107,8 @@ export const getContext = (callback = null) => {.then((input) => {context.input = input || null;return context;- })- .catch((err) => {- if (!callback) throw err;- callback(err);});-};---const exitWithError = (err, exitCode, message) => {- console.error(message);- console.error(err.stack || err);- process.exit(exitCode);+ return nodeifyPromise(promise, callback);};@@ -124,23 +117,38 @@ export const main = (userFunc) => {throw new Error('Handler function must be provided as a parameter');}+ // This is to enable unit tests where process.exit() is mocked and doesn't really exit the process+ // Note that mocked process.exit() might throw, so set exited flag before calling it to avoid confusion.+ let exited = false;+ const exitWithError = (err, exitCode, message) => {+ console.error(message);+ console.error(err.stack || err);+ exited = true;+ // console.log(`exitWithError: ${exitCode}`);+ process.exit(exitCode);+ };+try {newPromise().then(() => {return getContext();}).catch((err) => {- exitWithError(err, EXIT_CODES.ERROR_GETTING_INPUT, 'Failed to fetch act input');+ exitWithError(err, EXIT_CODES.ERROR_GETTING_INPUT, 'Failed to fetch act input:');}).then((context) => {- return userFunc(context);+ if (!exited) {+ return userFunc(context);+ }}).catch((err) => {- exitWithError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'User function threw an exception');+ if (!exited) {+ exitWithError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'User function threw an exception:');+ }}).then((userReturnValue) => {// Save output to the key-value store- if (userReturnValue) {+ if (!exited && userReturnValue) {const output = {body: JSON.stringify(userReturnValue),contentType: 'application/json',@@ -149,10 +157,14 @@ export const main = (userFunc) => {}}).catch((err) => {- exitWithError(err, EXIT_CODES.ERROR_SETTING_OUTPUT, 'Failed to save act output');+ if (!exited) {+ exitWithError(err, EXIT_CODES.ERROR_SETTING_OUTPUT, 'Failed to save act output:');+ }}).then(() => {- process.exit(EXIT_CODES.SUCCESS);+ if (!exited) {+ process.exit(EXIT_CODES.SUCCESS);+ }});} catch (err) {// This can happen e.g. if there's no Promise dependencydiff --git a/src/crawler.js b/src/crawler.jsdeleted file mode 100644index a2c8e82d7..000000000--- a/src/crawler.js+++ /dev/null@@ -1,75 +0,0 @@-/*-TODO: delete this-import _ from 'underscore';--import { newPromise, objectToQueryString } from './utils';--// TODO: please use requestPromise from utils-const request = require('request-promise');--const BASE_URL = 'https://api.apifier.com/v1';--let defaultUserId;-let defaultToken;--export const setDefaultToken = (newDefaultToken) => {- defaultToken = newDefaultToken;-};--export const setDefaultUserId = (newDefaultUserId) => {- defaultUserId = newDefaultUserId;-};--export const getAllCrawlers = (params, userId, token) => {- params = params || {};- params.token = token || defaultToken;- userId = userId || defaultUserId;-- const queryString = objectToQueryString(params);- const requestParams = {- url: `${BASE_URL}/${userId}/crawlers${queryString}`,- json: true,- resolveWithFullResponse: true,- };-- return newPromise().then(() => (request- .get(requestParams)))- .then((response) => {- const crawlers = response.body;- console.log(`crawlers: ${crawlers}`);- // checkPaginationHeaders(response, params, crawlers.length);- // checkResponse(response);- // if (!suppressSchemaCheck) crawlers.map(crawler => schemaValidator.validate('CrawlerShort', crawler));-- return crawlers;- });-};--export const startCrawler = (crawlerId, params, userId, token) => {- params = params || {};- params.token = token || defaultToken;- userId = userId || defaultUserId;-- const queryString = objectToQueryString(params);- const requestParams = {- url: `${BASE_URL}/${userId}/crawlers/${crawlerId}/execute${queryString}`,- json: true,- resolveWithFullResponse: true,- };-- return newPromise().then(() => (request- .post(requestParams)))- .then((response) => {- // checkResponse(response, 201);- const execution = response.body;-- if (!_.isObject(execution)) return reject('Unknown error of api Start Crawler API request');- if (execution.type === 'CRAWLING_PROCESS_QUOTA_EXCEEDED') return reject('CRAWLING_PROCESS_QUOTA_EXCEEDED');-- console.log(`execution: ${execution}`);- // if (!suppressSchemaCheck) schemaValidator.validate('CrawlerExecution', execution);-- return execution;- });-};-*/diff --git a/src/index.js b/src/index.jsindex 9a62cf7f6..aeb6c52b4 100644--- a/src/index.js+++ b/src/index.js@@ -1,18 +1,15 @@import { setPromisesDependency, getPromisesDependency } from './utils';-import { main, readyFreddy } from './actor';-import openKeyValueStore from './key-value-store';-import { setDefaultToken, setDefaultUserId, getAllCrawlers, startCrawler } from './crawler';+import { main, readyFreddy, getContext, getInput, setOutput, apifyClient } from './actor';const Apifier = {main,+ getContext,+ getInput,+ setOutput,readyFreddy,setPromisesDependency,getPromisesDependency,- openKeyValueStore,- setDefaultToken,- setDefaultUserId,- getAllCrawlers,- startCrawler,+ client: apifyClient,};// export this way so that we can import using:diff --git a/src/key-value-store.js b/src/key-value-store.jsdeleted file mode 100644index 9aa766316..000000000--- a/src/key-value-store.js+++ /dev/null@@ -1,169 +0,0 @@-import _ from 'underscore';-import { requestPromise, objectToQueryString, arrays2object } from './utils';--const methods = {- getStore: (baseUrl, storeId) => requestPromise({- url: `${baseUrl}/${storeId}`,- json: true,- method: 'GET',- }),-- drop: (baseUrl, storeId) => requestPromise({- url: `${baseUrl}/${storeId}`,- json: true,- method: 'DELETE',- }),-- get: (baseUrl, storeId, recordKey) => requestPromise({- url: `${baseUrl}/${storeId}/records/${recordKey}`,- json: true,- method: 'GET',- }),-- put: (baseUrl, storeId, recordKey, body, contentType) => requestPromise({- url: `${baseUrl}/${storeId}/records/${recordKey}`,- json: true,- method: 'PUT',- body,- headers: {- 'Content-Type': contentType,- },- }),-- delete: (baseUrl, storeId, recordKey) => requestPromise({- url: `${baseUrl}/${storeId}/records/${recordKey}`,- json: true,- method: 'DELETE',- }),-- keys: (baseUrl, storeId, exclusiveStartKey, count) => requestPromise({- url: `${baseUrl}/${storeId}/records${objectToQueryString({ exclusiveStartKey, count })}`,- json: true,- method: 'GET',- }),-- map: (baseUrl, storeId, exclusiveStartKey, count, callback) => {- return methods- .toArray(baseUrl, storeId, exclusiveStartKey, count)- .then(values => values.map(callback));- },-- mapObject: (baseUrl, storeId, exclusiveStartKey, count, callback) => {- return methods- .toObject(baseUrl, storeId, exclusiveStartKey, count)- .then(obj => _.mapObject(obj, callback));- },-- mapKeys: (baseUrl, storeId, exclusiveStartKey, count, callback) => {- return methods- .keys(baseUrl, storeId, exclusiveStartKey, count)- .then(keys => keys.map(callback));- },-- forEach: (baseUrl, storeId, exclusiveStartKey, count, callback) => {- return methods- .toObject(baseUrl, storeId, exclusiveStartKey, count)- .then((values) => {- _.forEach(values, callback);- });- },-- forEachKey: (baseUrl, storeId, exclusiveStartKey, count, callback) => {- return methods- .keys(baseUrl, storeId, exclusiveStartKey, count)- .then(keys => keys.forEach(callback));- },-- toObject: (baseUrl, storeId, exclusiveStartKey, count) => {- return methods- .keys(baseUrl, storeId, exclusiveStartKey, count)- .then((keys) => {- const getPromises = keys.map(key => methods.get(baseUrl, storeId, key));-- return Promise- .all(getPromises)- .then(values => arrays2object(keys, values));- });- },-- toArray: (baseUrl, storeId, exclusiveStartKey, count) => {- return methods- .keys(baseUrl, storeId, exclusiveStartKey, count)- .then((keys) => {- const getPromises = keys.map(key => methods.get(baseUrl, storeId, key));-- return Promise.all(getPromises);- });- },-- reduce: (baseUrl, storeId, exclusiveStartKey, count, callback, initial) => {- return methods- .keys(baseUrl, storeId, exclusiveStartKey, count)- .then((keys) => {- const getPromises = keys.map(key => methods.get(baseUrl, storeId, key));-- const reducer = (carry, currentVal, currentIndex) => {- return callback(carry, currentVal, keys[currentIndex]);- };-- return Promise- .all(getPromises)- .then(values => values.reduce(reducer, initial));- });- },-};--/**- * Gets or creates a key-value store and returns an object representing it.- * There are 3 possible initializations (performed in this order):- *- * 1) If `options.storeId` is provided then store with this id gets opened.- * 2) If `options.name`, `options.token` and one of `options.ownerUserId` and `options.ownerUser`- * are provided then store with given name gets opened or created if not exists.- * 3) If ENV.APIFY_ACT_RUN_ID exists then store with this id (ie. store belonging to this act run)- * gets opened.- */-const createOrGetStore = (baseUrl, options) => {- if (options.storeId) return methods.getStore(baseUrl, options.storeId);-- if ((options.ownerUserId || options.ownerUser) && options.token && options.name) {- return requestPromise({- url: baseUrl,- json: true,- method: 'POST',- body: _.pick(options, 'ownerUserId', 'ownerUser', 'token', 'name'),- });- }-- if (process.env.APIFY_ACT_RUN_ID) return methods.getStore(baseUrl, process.env.APIFY_ACT_RUN_ID);-- throw new Error('Cannot identify the key-value store to open.');-};--/**- * Opens a key-value store and returns object containing preconfigured methods for that store.- * Possible options are:- * - protocol (http/https)- * - host- * - port- * - basePath- * - storeId- * - ownerUser- * - ownerUserId- * - token- * - name- */-export default (options = {}) => {- // TODO: Why is this default export? Not sure this is a good idea, it might be confusing- if (!_.isObject(options)) throw new Error('The `options` parameter must be an object.');-- // eslint-disable-next-line prefer-template- const baseUrl = (options.protocol || 'https')- + '://'- + (options.host || 'api.apifier.com')- + (options.port ? `:${options.port}` : '')- + (options.basePath || '/v2/key-value-stores');-- return createOrGetStore(baseUrl, options)- .then(store => _.mapObject(methods, method => _.partial(method, baseUrl, store._id)));-};diff --git a/src/utils.js b/src/utils.jsindex f8d4a673a..22881a2f6 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,5 +1,3 @@-import request from 'request';-import _ from 'underscore';let PromisesDependency = typeof Promise === 'function' ? Promise : null;@@ -38,46 +36,10 @@ export const newPromise = () => {throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');};-/**- * Parses simple map { a: 'aa', b: 'bb' } to query string ?a=aa&b=bb.- */-export const objectToQueryString = (object) => {- const query = _.chain(object)- .mapObject(val => encodeURIComponent(val))- .mapObject((val, key) => `${key}=${val}`)- .toArray()- .join('&');-- return query ? `?${query}` : '';-};--/**- * Promised version of request(options) function.- */-export const requestPromise = (options) => {- const method = _.isString(options.method) ? options.method.toLowerCase() : options.method;-- if (!method) throw new Error('"options.method" parameter must be provided');- if (!request[method]) throw new Error('"options.method" is not a valid http request method');-- return new PromisesDependency((resolve, reject) => {- // We have to use request[method]({ ... }) instead of request({ method, ... })- // to be able to mock request when unit testing requestPromise().- request[method](options, (error, response, body) => {- if (error) return reject(error);-- resolve(body);- });- });-};--/**- * Zips two arrays ['a1', 'a2', ... ] and ['b1', 'b2', ...] into map { a1: 'b1', a2: 'b2', ... }.- */-export const arrays2object = (keys, vals) => {- return keys.reduce((prev, val, index) => {- prev[val] = vals[index];-- return prev;- }, {});+export const nodeifyPromise = (promise, callback) => {+ if (!promise) throw new Error('The "promise" parameter must be provided.');+ if (!callback) return promise;+ promise+ .then(() => callback(context))+ .catch(err => callback(err));};diff --git a/test/actor.js b/test/actor.jsindex b1b2c540d..cc6073995 100644--- a/test/actor.js+++ b/test/actor.js@@ -1,8 +1,10 @@import fs from 'fs';+import _ from 'underscore';import { expect } from 'chai';+import sinon from 'sinon';import tmp from 'tmp';-import request from 'request';-import portastic from 'portastic';++/* global process */// TODO: override console.log() to test the error messages (now they are printed to console)@@ -11,8 +13,9 @@ const Apifier = process.env.TEST_BABEL_BUILD ? require('../build/index') : requiif (process.env.TEST_BABEL_BUILD) console.log('Running with TEST_BABEL_BUILD option');-/* global process */+process.test = '1243';+/*let freePorts = [];before(() => {// find free ports for testing@@ -24,6 +27,8 @@ before(() => {freePorts = ports;});});+const popFreePort = () => freePorts.pop();+*/// always restore original process.exit()const origProcessExit = process.exit;@@ -31,8 +36,6 @@ after(() => {process.exit = origProcessExit;});-const popFreePort = () => freePorts.pop();-const createWatchFile = () => {const tmpobj = tmp.fileSync();@@ -59,7 +62,7 @@ const testWatchFileWillBecomeEmpty = (path, waitMillis) => {});};-+/*const testMain = (method, bodyRaw, contentType, userFunc, expectedExitCode = 0) => {const port = popFreePort();process.env.APIFY_INTERNAL_PORT = port;@@ -128,62 +131,284 @@ const testMain = (method, bodyRaw, contentType, userFunc, expectedExitCode = 0)});});};+*/++/**+ * Helper function that enables testing of Apifier.main()+ * @return A promise+ */+const testMain = ({ userFunc, context, exitCode, mockInputException }) => {+ // Mock process.exit() to check exit code and prevent process exit+ const processMock = sinon.mock(process);+ const exitExpectation = processMock+ .expects('exit')+ .withExactArgs(exitCode)+ .once()+ .returns();++ // Mock Apifier.client.keyValueStores.getRecord() for input getter+ const kvStoresMock = sinon.mock(Apifier.client.keyValueStores);+ if (mockInputException) {+ kvStoresMock+ .expects('getRecord')+ .throws(mockInputException);+ } else if (context.defaultKeyValueStoreId) {+ kvStoresMock+ .expects('getRecord')+ .withExactArgs({+ storeId: context.defaultKeyValueStoreId,+ promise: Apifier.getPromisesDependency(),+ }, null)+ .returns(Promise.resolve(context.input))+ .once();+ }++ // Mock APIFY_ environment variables+ _.defaults(context, getEmptyContext());+ setContextToEnv(context);++ // TODO: mock Apifier.client.keyValueStores.putRecord() for output setter++ // Waits max 1000 millis for process.exit() mock to be called,+ // then restores mocked functions and verifies they were called correctly+ const grandFinale = (err) => {+ // console.log(`XXX: grand finale: ${err}`);+ return new Promise((resolve) => {+ const waitUntil = Date.now() + 1000;+ const intervalId = setInterval(() => {+ // console.log('test for exitExpectation.called');+ if (!exitExpectation.called && Date.now() < waitUntil) return;+ clearInterval(intervalId);+ // console.log(`exitExpectation.called: ${exitExpectation.called}`);+ resolve();+ }, 10);+ })+ .then(() => {+ // console.log('XXX: restore');+ processMock.restore();+ kvStoresMock.restore();+ if (err) throw err;++ processMock.verify();+ kvStoresMock.verify();+ });+ };++ return new Promise((resolve, reject) => {+ // Invoke main() function, the promise resolves after the user function is run+ // Note that if mockInputException is set, then user function will never get called!+ if (!mockInputException) {+ Apifier.main((realContext) => {+ try {+ expect(realContext).to.eql(context);+ // Wait for all tasks in Node.js event loop to finish+ resolve();+ } catch (err) {+ reject(err);+ return;+ }+ // Call user func to test other behavior (note that it can throw)+ if (userFunc) return userFunc(realContext);+ });+ } else {+ Apifier.main(() => {});+ resolve();+ }+ })+ .catch(grandFinale)+ .then(grandFinale);+};+++const getEmptyContext = () => {+ return {+ internalPort: null,+ actId: null,+ actRunId: null,+ startedAt: null,+ timeoutAt: null,+ defaultKeyValueStoreId: null,+ input: null,+ };+};++const setContextToEnv = (context) => {+ delete process.env.APIFY_INTERNAL_PORT;+ delete process.env.APIFY_ACT_ID;+ delete process.env.APIFY_ACT_RUN_ID;+ delete process.env.APIFY_STARTED_AT;+ delete process.env.APIFY_TIMEOUT_AT;+ delete process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID;++ if (context.internalPort) process.env.APIFY_INTERNAL_PORT = context.internalPort.toString();+ if (context.actId) process.env.APIFY_ACT_ID = context.actId;+ if (context.actRunId) process.env.APIFY_ACT_RUN_ID = context.actRunId;+ if (context.startedAt) process.env.APIFY_STARTED_AT = context.startedAt.toISOString();+ if (context.timeoutAt) process.env.APIFY_TIMEOUT_AT = context.timeoutAt.toISOString();+ if (context.defaultKeyValueStoreId) process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = context.defaultKeyValueStoreId;+};+describe('Apifier.getContext()', () => {+ it('works with null values', () => {+ const expectedContext = getEmptyContext();+ setContextToEnv(expectedContext);++ return Apifier.getContext()+ .then((context) => {+ expect(context).to.eql(expectedContext);+ });+ });++ it('works with with non-null values / no input', () => {+ const expectedContext = _.extend(getEmptyContext(), {+ internalPort: 12345,+ actId: 'test actId',+ actRunId: 'test actId',+ startedAt: new Date('2017-01-01'),+ timeoutAt: new Date(),+ defaultKeyValueStoreId: null,+ input: null,+ });+ setContextToEnv(expectedContext);++ return Apifier.getContext()+ .then((context) => {+ expect(context).to.eql(expectedContext);+ });+ });++ it('works with with non-null values / text input', () => {+ const expectedContext = {+ internalPort: 12345,+ actId: 'test actId',+ actRunId: 'test actId',+ startedAt: new Date('2017-01-01'),+ timeoutAt: new Date(),+ defaultKeyValueStoreId: 'test storeId',+ input: {+ body: 'test body',+ contentType: 'text/plain',+ },+ };+ setContextToEnv(expectedContext);++ const mock = sinon.mock(Apifier.client.keyValueStores);+ const expectation = mock.expects('getRecord');+ expectation+ .withExactArgs({+ storeId: expectedContext.defaultKeyValueStoreId,+ promise: Apifier.getPromisesDependency(),+ }, null)+ .once()+ .returns(Promise.resolve(expectedContext.input));++ return Apifier.getContext()+ .then((context) => {+ expect(context).to.eql(expectedContext);+ mock.restore();+ expectation.verify();+ })+ .catch((err) => {+ mock.restore();+ throw err;+ });+ });+});describe('Apifier.main()', () => {it('throws on invalid args', () => {- process.env.APIFY_INTERNAL_PORT = 1234;expect(() => {Apifier.main();}).to.throw(Error);});- it('throws on invalid env vars', () => {- const fn = () => {- Apifier.main(() => {});- };-- process.env.APIFY_INTERNAL_PORT = null;- expect(fn).to.throw(Error);-- process.env.APIFY_INTERNAL_PORT = '';- expect(fn).to.throw(Error);-- process.env.APIFY_INTERNAL_PORT = 0;- expect(fn).to.throw(Error);-- process.env.APIFY_INTERNAL_PORT = 65536;- expect(fn).to.throw(Error);+ it('works with simple user function', () => {+ return testMain({+ userFunc: () => {},+ context: {},+ exitCode: 0,+ });});- it('passes text/plain POST request', () => {- return testMain('POST', 'testxxx', 'text/plain');+ it('works with promised user function', () => {+ let called = false;+ return testMain({+ userFunc: () => {+ return new Promise((resolve) => {+ setTimeout(() => {+ called = true;+ // console.log('called = true');+ resolve();+ }, 20);+ });+ },+ context: {},+ exitCode: 0,+ })+ .then(() => {+ expect(called).to.eql(true);+ });});- it('passes application/json PUT request', () => {- return testMain('PUT', JSON.stringify({ abc: 123 }), 'application/json');+ it('gets input correctly', () => {+ const context = {+ defaultKeyValueStoreId: 'test storeId',+ input: {+ body: 'test body',+ contentType: 'text/plain',+ },+ };+ return testMain({+ userFunc: null,+ context,+ exitCode: 0,+ });});- it('passes raw POST request', () => {- return testMain('POST', new Buffer('somebinarydata'), 'image/png');+ it('sets output from simple user function', () => {+ // TODO});- it('passes empty GET request with application/json content type', () => {- return testMain('GET', null, 'application/json');+ it('sets output from promised user function', () => {+ // TODO});- it('passes empty GET request with no content type', () => {- return testMain('GET', null, null);+ it('on exception in simple user function the process exits with code 1001', () => {+ return testMain({+ userFunc: () => {+ throw new Error('Test exception I');+ },+ context: {},+ exitCode: 1001,+ });});- it('on exception exits process with code 1', () => {- const userFunc = () => {- throw new Error('Test exception');- };- return testMain('PUT', 'testxxx', 'text/plain', userFunc, 1);+ it('on exception in promised user function the process exits with code 1001', () => {+ return testMain({+ userFunc: () => {+ return new Promise((resolve) => {+ setTimeout(resolve, 20);+ })+ .then(() => {+ throw new Error('Text exception II');+ });+ },+ context: {},+ exitCode: 1001,+ });});- // TODO: test responses from act, exceptions etc. !+ it('on exception in getInput the process exits with code 1002', () => {+ return testMain({+ userFunc: null,+ context: {+ defaultKeyValueStoreId: 'test storeId',+ input: {},+ },+ exitCode: 1002,+ mockInputException: new Error('Text exception III'),+ });+ });});diff --git a/test/key-value-store.js b/test/key-value-store.jsdeleted file mode 100644index 073142a93..000000000--- a/test/key-value-store.js+++ /dev/null@@ -1,416 +0,0 @@-import sinon from 'sinon';-import _ from 'underscore';-import { expect } from 'chai';-import * as utils from '../src/utils';--// NOTE: use require() here because this is how its done in acts-const Apifier = process.env.TEST_BABEL_BUILD ? require('../build/index') : require('../src/index');--const options = {- protocol: 'http',- host: 'myhost',- basePath: '/mypath',- port: 80,-};--describe('Key value store', () => {- let requestPromiseMock;-- const requestExpectOne = (arg, result) => {- requestPromiseMock- .expects('requestPromise')- .once()- .withArgs(arg)- .returns(Promise.resolve(result));- };-- before(() => {- requestPromiseMock = sinon.mock(utils, 'requestPromise');- });-- after(() => {- requestPromiseMock.verify();- requestPromiseMock.restore();- });-- describe('Key value store opening methods work', () => {- it('should work with ENV variable', () => {- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id',- }, {- _id: 'some-id',- });-- process.env.APIFY_ACT_RUN_ID = 'some-id';-- return Apifier- .openKeyValueStore(options)- .then(() => {- delete process.env.APIFY_ACT_RUN_ID;- });- });-- it('should work with storeId', () => {- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id-2',- }, {- _id: 'some-id',- });-- return Apifier.openKeyValueStore(Object.assign({}, options, { storeId: 'some-id-2' }));- });-- it('should work with user ID and credentials', () => {- requestExpectOne({- body: { name: 'somename', ownerUserId: 'someid', token: 'sometoken' },- json: true,- method: 'POST',- url: 'http://myhost:80/mypath',- }, {- _id: 'some-id',- });-- return Apifier.openKeyValueStore(Object.assign({}, options, {- ownerUserId: 'someid',- token: 'sometoken',- name: 'somename',- }));- });-- it('should work with username and credentials', () => {- requestExpectOne({- body: { name: 'somename', ownerUser: 'someusername', token: 'sometoken' },- json: true,- method: 'POST',- url: 'http://myhost:80/mypath',- }, {- _id: 'some-id',- });-- return Apifier.openKeyValueStore(Object.assign({}, options, {- ownerUser: 'someusername',- token: 'sometoken',- name: 'somename',- }));- });- });-- describe('Key value store REST methods work', () => {- let store;-- beforeEach(() => {- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id',- }, {- _id: 'some-id',- });-- const optionsWithId = Object.assign({}, options, { storeId: 'some-id' });-- return Apifier- .openKeyValueStore(optionsWithId)- .then((newStore) => {- store = newStore;- });- });-- it('getStore() works', () => {- const expected = { _id: 'some-id', aaa: 'bbb' };-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id',- }, expected);-- return store- .getStore()- .then(given => expect(given).to.be.eql(expected));- });-- it('drop() works', () => {- requestExpectOne({- json: true,- method: 'DELETE',- url: 'http://myhost:80/mypath/some-id',- });-- return store.drop();- });-- it('get() works', () => {- const expected = 'sometext';-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records/someId',- }, expected);-- return store- .get('someId')- .then(given => expect(given).to.be.eql(expected));- });-- it('put() works', () => {- requestExpectOne({- body: 'someValue',- headers: { 'Content-Type': 'application/json' },- json: true,- method: 'PUT',- url: 'http://myhost:80/mypath/some-id/records/someKey',- });-- return store.put('someKey', 'someValue', 'application/json');- });-- it('delete() works', () => {- requestExpectOne({- json: true,- method: 'DELETE',- url: 'http://myhost:80/mypath/some-id/records/someKey',- });-- return store.delete('someKey');- });-- it('keys() works', () => {- const expected = ['key1', 'key2', 'key3'];-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=10',- }, expected);-- return store- .keys('fromKey', 10)- .then(given => expect(given).to.be.eql(expected));- });- });-- describe('Key value store iteration methods work', () => {- let store;-- beforeEach(() => {- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id',- }, {- _id: 'some-id',- });-- const optionsWithId = Object.assign({}, options, { storeId: 'some-id' });-- return Apifier- .openKeyValueStore(optionsWithId)- .then((newStore) => {- store = newStore;- });- });-- it('forEach() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const values = ['val1', 'val2', 'val3', 'val4'];- const expected = utils.arrays2object(keys, values);-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');-- const result = {};-- return store- .forEach('fromKey', 5, (value, key) => {- result[key] = value;- })- .then((response) => {- expect(response).to.be.eql(undefined);- expect(result).to.be.eql(expected);- });- });-- it('forEachKey() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const expected = utils.arrays2object(_.range(0, 4), keys);-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);-- const result = {};-- return store- .forEachKey('fromKey', 5, (key, index) => {- result[index] = key;- })- .then((response) => {- expect(response).to.be.eql(undefined);- expect(result).to.be.eql(expected);- });- });-- it('map() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const values = ['val1', 'val2', 'val3', 'val4'];- const valuesModif = ['val1XXX', 'val2XXX', 'val3XXX', 'val4XXX'];-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');-- return store- .map('fromKey', 5, (value, key) => {- const index = values.indexOf(value);-- expect(key).to.be.eql(index);-- return `${value}XXX`;- })- .then((response) => {- expect(response).to.be.eql(valuesModif);- });- });-- it('mapKeys() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const keysModif = ['key1XXX', 'key2XXX', 'key3XXX', 'key4XXX'];-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);-- return store- .mapKeys('fromKey', 5, (key, index) => {- expect(index).to.be.eql(keys.indexOf(key));-- return `${key}XXX`;- })- .then((response) => {- expect(response).to.be.eql(keysModif);- });- });-- it('mapObject() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const values = ['val1', 'val2', 'val3', 'val4'];- const valuesModif = ['val1XXX', 'val2XXX', 'val3XXX', 'val4XXX'];- const expected = utils.arrays2object(keys, valuesModif);-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');-- return store- .mapObject('fromKey', 5, (value, key) => {- const index = values.indexOf(value);-- expect(key).to.be.eql(keys[index]);-- return `${value}XXX`;- })- .then((response) => {- expect(response).to.be.eql(expected);- });- });-- it('reduce() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const values = ['val1', 'val2', 'val3', 'val4'];- const expected = utils.arrays2object(keys, values);- expected.initialKey = 'initialValue';-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');-- return store- .reduce('fromKey', 5, (carry, currentVal, currentIndex) => {- carry[currentIndex] = currentVal;-- return carry;- }, { initialKey: 'initialValue' })- .then((response) => {- expect(response).to.be.eql(expected);- });- });-- it('toObject() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const values = ['val1', 'val2', 'val3', 'val4'];- const expected = utils.arrays2object(keys, values);-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');-- return store- .toObject('fromKey', 5)- .then((response) => {- expect(response).to.be.eql(expected);- });- });-- it('toArray() works', () => {- const keys = ['key1', 'key2', 'key3', 'key4'];- const values = ['val1', 'val2', 'val3', 'val4'];-- requestExpectOne({- json: true,- method: 'GET',- url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',- }, keys);- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');- requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');-- return store- .toArray('fromKey', 5)- .then((response) => {- expect(response).to.be.eql(values);- });- });- });-});diff --git a/test/utils.js b/test/utils.jsindex 7eb80fd6b..2a7e0ab74 100644--- a/test/utils.js+++ b/test/utils.js@@ -32,97 +32,6 @@ describe('Apifier.xxxPromisesDependency()', () => {});});-describe('utils.objectToQueryString()', () => {- it('should create correct query string out of a plain object', () => {- const obj = {- val1: 'something',- val2: 'else',- };- expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something&val2=else');- });-- it('should encode URI components', () => {- const obj = {- val1: 'something+else',- 'val&?/': 'ěščřžýá',- };-- expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something%2Belse&val%26%3F%2F=%C4%9B%C5%A1%C4%8D%C5%99%C5%BE%C3%BD%C3%A1');- });-- it('handles empty args', () => {- expect(utils.objectToQueryString({})).to.be.eql('');- expect(utils.objectToQueryString(null)).to.be.eql('');- expect(utils.objectToQueryString(undefined)).to.be.eql('');- });-});--describe('utils.requestPromise()', () => {- it('works as expected when request succeeds', () => {- const method = 'DELETE';- const opts = { method, foo: 'bar' };- const expected = { foo: 'something', bar: 123 };-- const stub = sinon- .stub(request, method.toLowerCase())- .callsFake((passedOpts, callback) => {- expect(passedOpts).to.be.eql(opts);- callback(null, {}, expected);- });-- return utils- .requestPromise(opts)- .then((response) => {- expect(response).to.be.eql(expected);- stub.restore();- });- });-- it('works as expected when request fails', () => {- const method = 'POST';- const opts = { method, foo: 'bar' };+// TODO: test nodeifyPromise() !!!- const stub = sinon- .stub(request, method.toLowerCase())- .callsFake((passedOpts, callback) => {- expect(passedOpts).to.be.eql(opts);- callback(new Error('some-error'));- });-- return utils- .requestPromise(opts)- .then(() => {- throw new Error('Error not catched!!!');- }, (err) => {- expect(err.message).to.be.eql('some-error');- stub.restore();- });- });-- it('fails when method parameter is not provided', () => {- let hasFailed = false;-- try {- utils.requestPromise({ method: null });- } catch (err) {- expect(err.message).to.be.eql('"options.method" parameter must be provided');- hasFailed = true;- }-- expect(hasFailed).to.be.eql(true);- });-- it('fails when request[method] doesn\'t exist', () => {- let hasFailed = false;-- try {- utils.requestPromise({ method: 'something' });- } catch (err) {- expect(err.message).to.be.eql('"options.method" is not a valid http request method');- hasFailed = true;- }-- expect(hasFailed).to.be.eql(true);- });-});