Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Implementing key value store sdk
Marek Trunkat4490bf68622e
diff --git a/.eslintrc.json b/.eslintrc.jsonindex a9cdec42a..fe91d93e8 100644--- a/.eslintrc.json+++ b/.eslintrc.json@@ -20,6 +20,7 @@"no-undef": 0,"func-names": 0,"import/prefer-default-export": 0,- "import/no-absolute-path": 0+ "import/no-absolute-path": 0,+ "import/no-extraneous-dependencies": ["error", { "devDependencies": ["**/test/*.js"] }]}}diff --git a/.gitignore b/.gitignoreindex 5ce4b4875..bbde28a78 100644--- a/.gitignore+++ b/.gitignore@@ -1,3 +1,4 @@+buildsnode_modules*.log*.piddiff --git a/build/index.js b/build/index.jsindex ff3b6f1a1..74f6baf60 100644--- a/build/index.js+++ b/build/index.js@@ -4,11 +4,18 @@ var _utils = require('./utils');var _actor = require('./actor');+var _keyValueStore = require('./key-value-store');++var _keyValueStore2 = _interopRequireDefault(_keyValueStore);++function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }+var Apifier = {main: _actor.main,heyIAmReady: _actor.heyIAmReady,setPromisesDependency: _utils.setPromisesDependency,- getPromisesDependency: _utils.getPromisesDependency+ getPromisesDependency: _utils.getPromisesDependency,+ openKeyValueStore: _keyValueStore2.default};// export this way so that we can import using:diff --git a/build/key-value-store.js b/build/key-value-store.jsnew file mode 100644index 000000000..65fd95309--- /dev/null+++ b/build/key-value-store.js@@ -0,0 +1,149 @@+'use strict';++var _underscore = require('underscore');++var _underscore2 = _interopRequireDefault(_underscore);++var _utils = require('./utils');++function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }++var methods = {+ getStore: function getStore(baseUrl, storeId) {+ return (0, _utils.requestPromise)({+ url: baseUrl + '/' + storeId,+ json: true,+ method: 'GET'+ });+ },++ deleteStore: function deleteStore(baseUrl, storeId) {+ return (0, _utils.requestPromise)({+ url: baseUrl + '/' + storeId,+ json: true,+ method: 'DELETE'+ });+ },++ getRecordsList: function getRecordsList(baseUrl, storeId) {+ return (0, _utils.requestPromise)({+ url: baseUrl + '/' + storeId + '/records',+ json: true,+ method: 'GET'+ });+ },++ getRecord: function getRecord(baseUrl, storeId, recordKey) {+ return (0, _utils.requestPromise)({+ url: baseUrl + '/' + storeId + '/records/' + recordKey,+ json: true,+ method: 'GET'+ });+ },++ putRecord: function putRecord(baseUrl, storeId, recordKey, body, contentType) {+ return (0, _utils.requestPromise)({+ url: baseUrl + '/' + storeId + '/records/' + recordKey,+ json: true,+ method: 'PUT',+ body: body,+ headers: {+ 'Content-Type': contentType+ }+ });+ },++ deleteRecord: function deleteRecord(baseUrl, storeId, recordKey) {+ return (0, _utils.requestPromise)({+ url: baseUrl + '/' + storeId + '/records/' + recordKey,+ json: true,+ method: 'DELETE'+ });+ }+};++/**+ * Get or creates key value store and returns it's object.+ * There are 3 possible initializations:+ *+ * 1) If `options.storeId` is provided then store with this id gets opened.+ * 2) If `options.name`, `options.token` and one of `options.ownerUserId` and `options.ownerUser`+ * are provided then store with given name gets opened or created if not exists.+ * 3) If ENV.APIFY_ACT_RUN_ID exists then store with this id (ie. store belonging to this act run)+ * gets opened.+ */+var createOrGetStore = function createOrGetStore(baseUrl, options) {+ if (options.storeId) return methods.getStore(baseUrl, options.storeId);++ if ((options.ownerUserId || options.ownerUser) && options.token && options.name) {+ return (0, _utils.requestPromise)({+ url: baseUrl,+ json: true,+ method: 'POST',+ body: _underscore2.default.pick(options, 'ownerUserId', 'ownerUser', 'token', 'name')+ });+ }++ if (process.env.APIFY_ACT_RUN_ID) return methods.getStore(baseUrl, process.env.APIFY_ACT_RUN_ID);++ throw new Error('Error: cannot identify store via storeId, credentials or ENV variable.');+};++/**+ * Opens key value store and returns object containing preconfigured methods for that store.+ * Possible options are:+ * - protocol (http/https)+ * - host+ * - port+ * - basePath+ * - storeId+ * - ownerUser+ * - ownerUserId+ * - token+ * - name+ */+// export default+var openKeyValueStore = function openKeyValueStore() {+ var options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};++ if (!_underscore2.default.isObject(options)) throw new Error('Error: `options` must be object or empty.');++ // eslint-disable-next-line prefer-template+ var baseUrl = (options.protocol || 'https') + '://' + (options.host || 'api.apifier.com') + (options.port ? ':' + options.port : '') + (options.basePath || '/v2/key-value-stores');++ return createOrGetStore(baseUrl, options).then(function (store) {+ return _underscore2.default.mapObject(methods, function (method) {+ return _underscore2.default.partial(method, baseUrl, store._id);+ });+ });+};++var options = {+ protocol: 'http',+ host: 'localhost',+ port: '3300',+ basePath: '/api/v2/key-value-stores',+ storeId: '9mNnMEBm3LCakrgw3'+};++var options2 = {+ protocol: 'http',+ host: 'localhost',+ port: '3300',+ basePath: '/api/v2/key-value-stores',+ ownerUser: 'mtrunkat',+ token: '4AqtcLL4aQCtcWj9BSoLLACwJ',+ name: 'new-store'+};++openKeyValueStore(options2).then(function (store) {+ return store.getStore().then(function (out) {+ return console.log(out);+ }).then(function () {+ return store.putRecord('xxx', '{ "a": 2 }', 'application/json; charset=utf-8');+ }).then(function () {+ return store.getRecord('xxx');+ }).then(function (out) {+ return console.log(out);+ });+});\ No newline at end of filediff --git a/build/utils.js b/build/utils.jsindex 978968c84..0ef900d98 100644--- a/build/utils.js+++ b/build/utils.js@@ -3,12 +3,17 @@Object.defineProperty(exports, "__esModule", {value: true});+exports.requestPromise = exports.newPromise = exports.getPromisesDependency = exports.setPromisesDependency = undefined;+var _request = require('request');-var promisesDependency = typeof Promise === 'function' ? Promise : null;+var _request2 = _interopRequireDefault(_request);-// TODO: add methods to override console.log() and console.error(), add unit tests for that!+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }++var PromisesDependency = typeof Promise === 'function' ? Promise : null;+// TODO: add methods to override console.log() and console.error(), add unit tests for that!/*** Sets the promise dependency the SDK will use wherever Promises are returned.@@ -17,14 +22,14 @@ var promisesDependency = typeof Promise === 'function' ? Promise : null;*/var setPromisesDependency = exports.setPromisesDependency = function setPromisesDependency(dep) {if (dep !== null && typeof dep !== 'function') throw new Error('The "dep" parameter must be a function');- promisesDependency = dep;+ PromisesDependency = dep;};/*** Gets the promise dependency set by `Apifier.setPromisesDependency`.*/var getPromisesDependency = exports.getPromisesDependency = function getPromisesDependency() {- return promisesDependency;+ return PromisesDependency;};/**@@ -33,12 +38,25 @@ var getPromisesDependency = exports.getPromisesDependency = function getPromises* @return {*}*/var newPromise = exports.newPromise = function newPromise() {- if (promisesDependency) {- if (typeof promisesDependency.resolve !== 'function') {+ if (PromisesDependency) {+ if (typeof PromisesDependency.resolve !== 'function') {throw new Error('The promise dependency set using Apifier.setPromisesDependency() does not define resolve() function.');}- return promisesDependency.resolve();+ return PromisesDependency.resolve();}if (typeof Promise === 'function') return Promise.resolve();throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');+};++/**+ * Promised version of request(options) function.+ */+var requestPromise = exports.requestPromise = function requestPromise(options) {+ return new PromisesDependency(function (resolve, reject) {+ (0, _request2.default)(options, function (error, response, body) {+ if (error) return reject(error);++ resolve(body);+ });+ });};\ No newline at end of filediff --git a/package.json b/package.jsonindex cfc270771..7efe32e4f 100644--- a/package.json+++ b/package.json@@ -38,7 +38,9 @@},"dependencies": {"body-parser": "^1.17.1",- "express": "^4.15.2"+ "express": "^4.15.2",+ "request": "^2.81.0",+ "underscore": "^1.8.3"},"devDependencies": {"babel-cli": "^6.6.5",@@ -56,8 +58,10 @@"eslint-plugin-react": "^6.10.3","isparta": "^4.0.0","mocha": "^3.2.0",- "tmp": "^0.0.31",+ "portastic": "^1.0.1","request": "^2.81.0",- "portastic": "^1.0.1"+ "sinon": "^2.2.0",+ "sinon-stub-promise": "^4.0.0",+ "tmp": "^0.0.31"}}diff --git a/src/index.js b/src/index.jsindex 7450b97e4..ae4c9e5fe 100644--- a/src/index.js+++ b/src/index.js@@ -1,11 +1,14 @@import { setPromisesDependency, getPromisesDependency } from './utils';import { main, heyIAmReady } from './actor';+import openKeyValueStore from './key-value-store';+const Apifier = {main,heyIAmReady,setPromisesDependency,getPromisesDependency,+ openKeyValueStore,};diff --git a/src/key-value-store.js b/src/key-value-store.jsnew file mode 100644index 000000000..9c081ed2d--- /dev/null+++ b/src/key-value-store.js@@ -0,0 +1,168 @@+import _ from 'underscore';+import { requestPromise, objectToQueryString, arrays2object } from './utils';++const methods = {+ getStore: (baseUrl, storeId) => requestPromise({+ url: `${baseUrl}/${storeId}`,+ json: true,+ method: 'GET',+ }),++ drop: (baseUrl, storeId) => requestPromise({+ url: `${baseUrl}/${storeId}`,+ json: true,+ method: 'DELETE',+ }),++ get: (baseUrl, storeId, recordKey) => requestPromise({+ url: `${baseUrl}/${storeId}/records/${recordKey}`,+ json: true,+ method: 'GET',+ }),++ put: (baseUrl, storeId, recordKey, body, contentType) => requestPromise({+ url: `${baseUrl}/${storeId}/records/${recordKey}`,+ json: true,+ method: 'PUT',+ body,+ headers: {+ 'Content-Type': contentType,+ },+ }),++ delete: (baseUrl, storeId, recordKey) => requestPromise({+ url: `${baseUrl}/${storeId}/records/${recordKey}`,+ json: true,+ method: 'DELETE',+ }),++ keys: (baseUrl, storeId, exclusiveStartKey, count) => requestPromise({+ url: `${baseUrl}/${storeId}/records${objectToQueryString({ exclusiveStartKey, count })}`,+ json: true,+ method: 'GET',+ }),++ map: (baseUrl, storeId, exclusiveStartKey, count, callback) => {+ return methods+ .toArray(baseUrl, storeId, exclusiveStartKey, count)+ .then(values => values.map(callback));+ },++ mapObject: (baseUrl, storeId, exclusiveStartKey, count, callback) => {+ return methods+ .toObject(baseUrl, storeId, exclusiveStartKey, count)+ .then(obj => _.mapObject(obj, callback));+ },++ mapKeys: (baseUrl, storeId, exclusiveStartKey, count, callback) => {+ return methods+ .keys(baseUrl, storeId, exclusiveStartKey, count)+ .then(keys => keys.map(callback));+ },++ forEach: (baseUrl, storeId, exclusiveStartKey, count, callback) => {+ return methods+ .toObject(baseUrl, storeId, exclusiveStartKey, count)+ .then((values) => {+ _.forEach(values, callback);+ });+ },++ forEachKey: (baseUrl, storeId, exclusiveStartKey, count, callback) => {+ return methods+ .keys(baseUrl, storeId, exclusiveStartKey, count)+ .then(keys => keys.forEach(callback));+ },++ toObject: (baseUrl, storeId, exclusiveStartKey, count) => {+ return methods+ .keys(baseUrl, storeId, exclusiveStartKey, count)+ .then((keys) => {+ const getPromises = keys.map(key => methods.get(baseUrl, storeId, key));++ return Promise+ .all(getPromises)+ .then(values => arrays2object(keys, values));+ });+ },++ toArray: (baseUrl, storeId, exclusiveStartKey, count) => {+ return methods+ .keys(baseUrl, storeId, exclusiveStartKey, count)+ .then((keys) => {+ const getPromises = keys.map(key => methods.get(baseUrl, storeId, key));++ return Promise.all(getPromises);+ });+ },++ reduce: (baseUrl, storeId, exclusiveStartKey, count, callback, initial) => {+ return methods+ .keys(baseUrl, storeId, exclusiveStartKey, count)+ .then((keys) => {+ const getPromises = keys.map(key => methods.get(baseUrl, storeId, key));++ const reducer = (carry, currentVal, currentIndex) => {+ return callback(carry, currentVal, keys[currentIndex]);+ };++ return Promise+ .all(getPromises)+ .then(values => values.reduce(reducer, initial));+ });+ },+};++/**+ * Get or creates key value store and returns it's object.+ * There are 3 possible initializations:+ *+ * 1) If `options.storeId` is provided then store with this id gets opened.+ * 2) If `options.name`, `options.token` and one of `options.ownerUserId` and `options.ownerUser`+ * are provided then store with given name gets opened or created if not exists.+ * 3) If ENV.APIFY_ACT_RUN_ID exists then store with this id (ie. store belonging to this act run)+ * gets opened.+ */+const createOrGetStore = (baseUrl, options) => {+ if (options.storeId) return methods.getStore(baseUrl, options.storeId);++ if ((options.ownerUserId || options.ownerUser) && options.token && options.name) {+ return requestPromise({+ url: baseUrl,+ json: true,+ method: 'POST',+ body: _.pick(options, 'ownerUserId', 'ownerUser', 'token', 'name'),+ });+ }++ if (process.env.APIFY_ACT_RUN_ID) return methods.getStore(baseUrl, process.env.APIFY_ACT_RUN_ID);++ throw new Error('Error: cannot identify store via storeId, credentials or ENV variable.');+};++/**+ * Opens key value store and returns object containing preconfigured methods for that store.+ * Possible options are:+ * - protocol (http/https)+ * - host+ * - port+ * - basePath+ * - storeId+ * - ownerUser+ * - ownerUserId+ * - token+ * - name+ */+export default (options = {}) => {+ if (!_.isObject(options)) throw new Error('Error: `options` must be object or empty.');++ // eslint-disable-next-line prefer-template+ const baseUrl = (options.protocol || 'https')+ + '://'+ + (options.host || 'api.apifier.com')+ + (options.port ? `:${options.port}` : '')+ + (options.basePath || '/v2/key-value-stores');++ return createOrGetStore(baseUrl, options)+ .then(store => _.mapObject(methods, method => _.partial(method, baseUrl, store._id)));+};diff --git a/src/utils.js b/src/utils.jsindex 750334152..e761c32e2 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,10 +1,10 @@+import request from 'request';+import _ from 'underscore';--let promisesDependency = typeof Promise === 'function' ? Promise : null;+let PromisesDependency = typeof Promise === 'function' ? Promise : null;// TODO: add methods to override console.log() and console.error(), add unit tests for that!-/*** Sets the promise dependency the SDK will use wherever Promises are returned.* Passing `null` will force the SDK to use native Promises if they are available.@@ -12,29 +12,70 @@ let promisesDependency = typeof Promise === 'function' ? Promise : null;*/export const setPromisesDependency = (dep) => {if (dep !== null && typeof dep !== 'function') throw new Error('The "dep" parameter must be a function');- promisesDependency = dep;+ PromisesDependency = dep;};/*** Gets the promise dependency set by `Apifier.setPromisesDependency`.*/export const getPromisesDependency = () => {- return promisesDependency;+ return PromisesDependency;};-/*** Returns a result of `Promise.resolve()` using promise library set by `setPromisesDependency()`,* or using native promises, or throws if no native promises are available.* @return {*}*/export const newPromise = () => {- if (promisesDependency) {- if (typeof (promisesDependency.resolve) !== 'function') {+ if (PromisesDependency) {+ if (typeof (PromisesDependency.resolve) !== 'function') {throw new Error('The promise dependency set using Apifier.setPromisesDependency() does not define resolve() function.');}- return promisesDependency.resolve();+ return PromisesDependency.resolve();}if (typeof Promise === 'function') return Promise.resolve();throw new Error('Native promises are not available, please call Apifier.setPromisesDependency() to set a promise library.');};++/**+ * Parses simple map { a: 'aa', b: 'bb' } to query string ?a=aa&b=bb.+ */+export const objectToQueryString = (object) => {+ const query = _.chain(object)+ .mapObject(val => encodeURIComponent(val))+ .mapObject((val, key) => `${key}=${val}`)+ .toArray()+ .join('&');++ return query ? `?${query}` : '';+};++/**+ * Promised version of request(options) function.+ */+export const requestPromise = (options) => {+ const method = _.isString(options.method) ? options.method.toLowerCase() : options.method;++ if (!method) throw new Error('"options.method" parameter must be provided');+ if (!request[method]) throw new Error('"options.method" is not a valid http request method');++ return new PromisesDependency((resolve, reject) => {+ request[method](options, (error, response, body) => {+ if (error) return reject(error);++ resolve(body);+ });+ });+};++/**+ * Zips to arrays ['a1', 'a2', ... ] and ['b1', 'b2', ...] into map { a1: 'b1', a2: 'b2', ... }.+ */+export const arrays2object = (keys, vals) => {+ return keys.reduce((prev, val, index) => {+ prev[val] = vals[index];++ return prev;+ }, {});+};diff --git a/test/key-value-store.js b/test/key-value-store.jsnew file mode 100644index 000000000..073142a93--- /dev/null+++ b/test/key-value-store.js@@ -0,0 +1,416 @@+import sinon from 'sinon';+import _ from 'underscore';+import { expect } from 'chai';+import * as utils from '../src/utils';++// NOTE: use require() here because this is how its done in acts+const Apifier = process.env.TEST_BABEL_BUILD ? require('../build/index') : require('../src/index');++const options = {+ protocol: 'http',+ host: 'myhost',+ basePath: '/mypath',+ port: 80,+};++describe('Key value store', () => {+ let requestPromiseMock;++ const requestExpectOne = (arg, result) => {+ requestPromiseMock+ .expects('requestPromise')+ .once()+ .withArgs(arg)+ .returns(Promise.resolve(result));+ };++ before(() => {+ requestPromiseMock = sinon.mock(utils, 'requestPromise');+ });++ after(() => {+ requestPromiseMock.verify();+ requestPromiseMock.restore();+ });++ describe('Key value store opening methods work', () => {+ it('should work with ENV variable', () => {+ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id',+ }, {+ _id: 'some-id',+ });++ process.env.APIFY_ACT_RUN_ID = 'some-id';++ return Apifier+ .openKeyValueStore(options)+ .then(() => {+ delete process.env.APIFY_ACT_RUN_ID;+ });+ });++ it('should work with storeId', () => {+ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id-2',+ }, {+ _id: 'some-id',+ });++ return Apifier.openKeyValueStore(Object.assign({}, options, { storeId: 'some-id-2' }));+ });++ it('should work with user ID and credentials', () => {+ requestExpectOne({+ body: { name: 'somename', ownerUserId: 'someid', token: 'sometoken' },+ json: true,+ method: 'POST',+ url: 'http://myhost:80/mypath',+ }, {+ _id: 'some-id',+ });++ return Apifier.openKeyValueStore(Object.assign({}, options, {+ ownerUserId: 'someid',+ token: 'sometoken',+ name: 'somename',+ }));+ });++ it('should work with username and credentials', () => {+ requestExpectOne({+ body: { name: 'somename', ownerUser: 'someusername', token: 'sometoken' },+ json: true,+ method: 'POST',+ url: 'http://myhost:80/mypath',+ }, {+ _id: 'some-id',+ });++ return Apifier.openKeyValueStore(Object.assign({}, options, {+ ownerUser: 'someusername',+ token: 'sometoken',+ name: 'somename',+ }));+ });+ });++ describe('Key value store REST methods work', () => {+ let store;++ beforeEach(() => {+ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id',+ }, {+ _id: 'some-id',+ });++ const optionsWithId = Object.assign({}, options, { storeId: 'some-id' });++ return Apifier+ .openKeyValueStore(optionsWithId)+ .then((newStore) => {+ store = newStore;+ });+ });++ it('getStore() works', () => {+ const expected = { _id: 'some-id', aaa: 'bbb' };++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id',+ }, expected);++ return store+ .getStore()+ .then(given => expect(given).to.be.eql(expected));+ });++ it('drop() works', () => {+ requestExpectOne({+ json: true,+ method: 'DELETE',+ url: 'http://myhost:80/mypath/some-id',+ });++ return store.drop();+ });++ it('get() works', () => {+ const expected = 'sometext';++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records/someId',+ }, expected);++ return store+ .get('someId')+ .then(given => expect(given).to.be.eql(expected));+ });++ it('put() works', () => {+ requestExpectOne({+ body: 'someValue',+ headers: { 'Content-Type': 'application/json' },+ json: true,+ method: 'PUT',+ url: 'http://myhost:80/mypath/some-id/records/someKey',+ });++ return store.put('someKey', 'someValue', 'application/json');+ });++ it('delete() works', () => {+ requestExpectOne({+ json: true,+ method: 'DELETE',+ url: 'http://myhost:80/mypath/some-id/records/someKey',+ });++ return store.delete('someKey');+ });++ it('keys() works', () => {+ const expected = ['key1', 'key2', 'key3'];++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=10',+ }, expected);++ return store+ .keys('fromKey', 10)+ .then(given => expect(given).to.be.eql(expected));+ });+ });++ describe('Key value store iteration methods work', () => {+ let store;++ beforeEach(() => {+ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id',+ }, {+ _id: 'some-id',+ });++ const optionsWithId = Object.assign({}, options, { storeId: 'some-id' });++ return Apifier+ .openKeyValueStore(optionsWithId)+ .then((newStore) => {+ store = newStore;+ });+ });++ it('forEach() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const values = ['val1', 'val2', 'val3', 'val4'];+ const expected = utils.arrays2object(keys, values);++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');++ const result = {};++ return store+ .forEach('fromKey', 5, (value, key) => {+ result[key] = value;+ })+ .then((response) => {+ expect(response).to.be.eql(undefined);+ expect(result).to.be.eql(expected);+ });+ });++ it('forEachKey() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const expected = utils.arrays2object(_.range(0, 4), keys);++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);++ const result = {};++ return store+ .forEachKey('fromKey', 5, (key, index) => {+ result[index] = key;+ })+ .then((response) => {+ expect(response).to.be.eql(undefined);+ expect(result).to.be.eql(expected);+ });+ });++ it('map() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const values = ['val1', 'val2', 'val3', 'val4'];+ const valuesModif = ['val1XXX', 'val2XXX', 'val3XXX', 'val4XXX'];++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');++ return store+ .map('fromKey', 5, (value, key) => {+ const index = values.indexOf(value);++ expect(key).to.be.eql(index);++ return `${value}XXX`;+ })+ .then((response) => {+ expect(response).to.be.eql(valuesModif);+ });+ });++ it('mapKeys() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const keysModif = ['key1XXX', 'key2XXX', 'key3XXX', 'key4XXX'];++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);++ return store+ .mapKeys('fromKey', 5, (key, index) => {+ expect(index).to.be.eql(keys.indexOf(key));++ return `${key}XXX`;+ })+ .then((response) => {+ expect(response).to.be.eql(keysModif);+ });+ });++ it('mapObject() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const values = ['val1', 'val2', 'val3', 'val4'];+ const valuesModif = ['val1XXX', 'val2XXX', 'val3XXX', 'val4XXX'];+ const expected = utils.arrays2object(keys, valuesModif);++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');++ return store+ .mapObject('fromKey', 5, (value, key) => {+ const index = values.indexOf(value);++ expect(key).to.be.eql(keys[index]);++ return `${value}XXX`;+ })+ .then((response) => {+ expect(response).to.be.eql(expected);+ });+ });++ it('reduce() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const values = ['val1', 'val2', 'val3', 'val4'];+ const expected = utils.arrays2object(keys, values);+ expected.initialKey = 'initialValue';++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');++ return store+ .reduce('fromKey', 5, (carry, currentVal, currentIndex) => {+ carry[currentIndex] = currentVal;++ return carry;+ }, { initialKey: 'initialValue' })+ .then((response) => {+ expect(response).to.be.eql(expected);+ });+ });++ it('toObject() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const values = ['val1', 'val2', 'val3', 'val4'];+ const expected = utils.arrays2object(keys, values);++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');++ return store+ .toObject('fromKey', 5)+ .then((response) => {+ expect(response).to.be.eql(expected);+ });+ });++ it('toArray() works', () => {+ const keys = ['key1', 'key2', 'key3', 'key4'];+ const values = ['val1', 'val2', 'val3', 'val4'];++ requestExpectOne({+ json: true,+ method: 'GET',+ url: 'http://myhost:80/mypath/some-id/records?exclusiveStartKey=fromKey&count=5',+ }, keys);+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key1' }, 'val1');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key2' }, 'val2');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key3' }, 'val3');+ requestExpectOne({ json: true, method: 'GET', url: 'http://myhost:80/mypath/some-id/records/key4' }, 'val4');++ return store+ .toArray('fromKey', 5)+ .then((response) => {+ expect(response).to.be.eql(values);+ });+ });+ });+});diff --git a/test/utils.js b/test/utils.jsindex b97b12e2d..979673279 100644--- a/test/utils.js+++ b/test/utils.js@@ -1,5 +1,7 @@+import sinon from 'sinon';+import request from 'request';import { expect } from 'chai';-import { newPromise } from '../src/utils';+import * as utils from '../src/utils';// NOTE: use require() here because this how its done in actsconst Apifier = process.env.TEST_BABEL_BUILD ? require('../build/index') : require('../src/index');@@ -22,10 +24,99 @@ describe('Apifier.xxxPromisesDependency()', () => {it('should work as expected', () => {Apifier.setPromisesDependency(null);expect(Apifier.getPromisesDependency()).to.be.a('null');- expect(newPromise()).to.have.property('then');+ expect(utils.newPromise()).to.have.property('then');Apifier.setPromisesDependency(Promise);expect(Apifier.getPromisesDependency()).to.equal(Promise);- expect(newPromise()).to.have.property('then');+ expect(utils.newPromise()).to.have.property('then');+ });+});++describe('utils.objectToQueryString()', () => {+ it('should create correct query string out of a plain object', () => {+ const obj = {+ val1: 'something',+ val2: 'else',+ };++ expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something&val2=else');+ });++ it('should encode uri components', () => {+ const obj = {+ val1: 'something+else',+ val2: 'ěščřžýá',+ };++ expect(utils.objectToQueryString(obj)).to.be.eql('?val1=something%2Belse&val2=%C4%9B%C5%A1%C4%8D%C5%99%C5%BE%C3%BD%C3%A1');+ });+});++describe('utils.requestPromise()', () => {+ it('works as expected when request succeeds', () => {+ const method = 'DELETE';+ const opts = { method, foo: 'bar' };+ const expected = { foo: 'something', bar: 123 };++ const stub = sinon+ .stub(request, method.toLowerCase())+ .callsFake((passedOpts, callback) => {+ expect(passedOpts).to.be.eql(opts);+ callback(null, {}, expected);+ });++ return utils+ .requestPromise(opts)+ .then((response) => {+ expect(response).to.be.eql(expected);+ stub.restore();+ });+ });++ it('works as expected when request fails', () => {+ const method = 'POST';+ const opts = { method, foo: 'bar' };++ const stub = sinon+ .stub(request, method.toLowerCase())+ .callsFake((passedOpts, callback) => {+ expect(passedOpts).to.be.eql(opts);+ callback(new Error('some-error'));+ });++ return utils+ .requestPromise(opts)+ .then(() => {+ throw new Error('Error not catched!!!');+ }, (err) => {+ expect(err.message).to.be.eql('some-error');+ stub.restore();+ });+ });++ it('fails when method parameter is not provided', () => {+ let hasFailed = false;++ try {+ utils.requestPromise({ method: null });+ } catch (err) {+ expect(err.message).to.be.eql('"options.method" parameter must be provided');+ hasFailed = true;+ }++ expect(hasFailed).to.be.eql(true);+ });++ it('fails when request[method] doesn\'t exist', () => {+ let hasFailed = false;++ try {+ utils.requestPromise({ method: 'something' });+ } catch (err) {+ expect(err.message).to.be.eql('"options.method" is not a valid http request method');+ hasFailed = true;+ }++ expect(hasFailed).to.be.eql(true);});});