Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Renamed pushItem to pushData and allowed arrays

Jaroslav Hejlek61851a568615
diff --git a/src/actor.js b/src/actor.js
index bcb6beeb0..e49aa1b24 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -305,35 +305,35 @@ export const setValue = (key, value, options, callback = null) => {
* @ignore
* @memberof module:Apify
* @function
- * @description <p>Stores an item (object) in a dataset using the Apify API.
+ * @description <p>Stores data (object or array of objects) in a dataset using the Apify API.
* The function has no result, but throws on invalid args or other errors.</p>
- * <pre><code class="language-javascript">await Apify.pushItem(item);</code></pre>
+ * <pre><code class="language-javascript">await Apify.pushData(data);</code></pre>
* <p>
- * By default, the record is stored as is in default sequential store associated with this act.
+ * By default, the data is stored in default dataset associated with this act.
* </p>
* <p>
- * **IMPORTANT: Do not forget to use the `await` keyword when calling `Apify.pushItem()`,
- * otherwise the act process might finish before the record is stored!**
+ * **IMPORTANT: Do not forget to use the `await` keyword when calling `Apify.pushData()`,
+ * otherwise the act process might finish before the data is stored!**
* </p>
- * @param {Object} item Object containing data to by stored in the dataset
+ * @param {Object/Array} data Object or array of Objects containing data to by stored in the dataset
* @param {Function} [callback] Optional callback. Function returns a promise if not provided.
* @returns {Promise} Returns a promise if `callback` was not provided.
*/
-export const pushItem = (item, callback = null) => {
- if (!item || !_.isObject(item) || _.isArray(item)) throw new Error('The "item" parameter must be an object');
+export const pushData = (data, callback = null) => {
+ if (!data || (!_.isObject(data) && !_.isArray(data))) throw new Error('The "data" parameter must be an object or array');
if (callback && !_.isFunction(callback)) throw new Error('If provided then the "callback" parameter must be a function');
const promisePrototype = getPromisePrototype();
- let stringifiedItem;
+ let stringifiedData;
try {
// Format JSON to simplify debugging, the overheads with compression is negligible
- stringifiedItem = JSON.stringify(item, null, 2);
+ stringifiedData = JSON.stringify(data, null, 2);
} catch (e) {
- throw new Error(`The "item" parameter cannot be stringified to JSON: ${e.message}`);
+ throw new Error(`The "data" parameter cannot be stringified to JSON: ${e.message}`);
}
- if (stringifiedItem === undefined) {
- throw new Error('The "item" parameter cannot be stringified to JSON.');
+ if (stringifiedData === undefined) {
+ throw new Error('The "data" parameter cannot be stringified to JSON.');
}
const datasetId = getDefaultDatasetIdOrThrow();
@@ -341,7 +341,7 @@ export const pushItem = (item, callback = null) => {
const innerPromise = apifyClient.datasets.putItem({
datasetId,
promise: promisePrototype,
- data: item,
+ data,
});
diff --git a/src/index.js b/src/index.js
index efe88679b..d71a122ee 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,7 +1,7 @@
import EventEmitter from 'events';
import { setPromisesDependency, getPromisesDependency } from './utils';
-import { main, readyFreddy, getEnv, getValue, setValue, pushItem, apifyClient, call, getMemoryInfo } from './actor';
+import { main, readyFreddy, getEnv, getValue, setValue, pushData, apifyClient, call, getMemoryInfo } from './actor';
import { launchPuppeteer } from './puppeteer';
import { browse, launchWebDriver } from './webdriver';
@@ -13,7 +13,7 @@ const Apify = {
getEnv,
getValue,
setValue,
- pushItem,
+ pushData,
call,
readyFreddy,
setPromisesDependency,
diff --git a/test/actor.js b/test/actor.js
index 95017852a..f36b5e6c3 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -940,32 +940,31 @@ describe('Apify.setValue()', () => {
});
});
-describe('Apify.pushItem()', () => {
+describe('Apify.pushData()', () => {
it('throws on invalid args', () => {
process.env.APIFY_DEFAULT_DATASET_ID = '1234';
- const recordErrMsg = 'The "item" parameter must be an object';
- expect(() => { Apify.pushItem(); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushItem(''); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushItem(123); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushItem(true); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushItem(false); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushItem([]); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushItem(() => {}, () => {}); }).to.throw(Error, jsonErrMsg);
+ const recordErrMsg = 'The "data" parameter must be an object or array';
+ expect(() => { Apify.pushData(); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushData(''); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushData(123); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushData(true); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushData(false); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushData(() => {}, () => {}); }).to.throw(Error, jsonErrMsg);
const circularObj = {};
circularObj.xxx = circularObj;
- const jsonErrMsg = 'The "item" parameter cannot be stringified to JSON';
- expect(() => { Apify.pushItem(circularObj, null); }).to.throw(Error, jsonErrMsg);
+ const jsonErrMsg = 'The "data" parameter cannot be stringified to JSON';
+ expect(() => { Apify.pushData(circularObj, null); }).to.throw(Error, jsonErrMsg);
});
it('throws if APIFY_DEFAULT_DATASET_ID env var is not defined', () => {
const errMsg = 'The \'APIFY_DEFAULT_DATASET_ID\' environment variable is not defined';
process.env.APIFY_DEFAULT_DATASET_ID = '';
- expect(() => { Apify.pushItem({ something: 123 }); }).to.throw(Error, errMsg);
+ expect(() => { Apify.pushData({ something: 123 }); }).to.throw(Error, errMsg);
delete process.env.APIFY_DEFAULT_DATASET_ID;
- expect(() => { Apify.pushItem({ something: 123 }); }).to.throw(Error, errMsg);
+ expect(() => { Apify.pushData({ something: 123 }); }).to.throw(Error, errMsg);
});
it('supports both promises and callbacks (on success)', () => {
@@ -978,12 +977,12 @@ describe('Apify.pushItem()', () => {
return Promise.resolve()
.then(() => {
// test promise (no options)
- return Apify.pushItem({ someValue: 123 });
+ return Apify.pushData({ someValue: 123 });
})
.then(() => {
// test callback (no options)
return new Promise((resolve, reject) => {
- Apify.pushItem({ someValue: 123 }, (err) => {
+ Apify.pushData({ someValue: 123 }, (err) => {
if (err) return reject(err);
resolve();
});
@@ -1009,7 +1008,7 @@ describe('Apify.pushItem()', () => {
// Test promise
return Promise.resolve()
.then(() => {
- return Apify.pushItem({ someValue: 1 });
+ return Apify.pushData({ someValue: 1 });
})
.then(() => {
assert.fail();
@@ -1021,7 +1020,7 @@ describe('Apify.pushItem()', () => {
.then(() => {
// Test callback
return new Promise((resolve, reject) => {
- Apify.pushItem({ someValue: 1 }, (err) => {
+ Apify.pushData({ someValue: 1 }, (err) => {
if (err) return reject(err);
resolve();
});
@@ -1041,7 +1040,7 @@ describe('Apify.pushItem()', () => {
});
});
- it('correctly stores records', () => {
+ it('correctly stores items', () => {
const datasetId = 'mystore';
const value = { someValue: 123 };
@@ -1061,7 +1060,37 @@ describe('Apify.pushItem()', () => {
return Promise.resolve()
.then(() => {
- return Apify.pushItem(value);
+ return Apify.pushData(value);
+ })
+ .then(() => {
+ mock.verify();
+ })
+ .finally(() => {
+ mock.restore();
+ });
+ });
+
+ it('correctly stores items when provided as array', () => {
+ const datasetId = 'mystore';
+ const value = [{ someValue: 123 }, { someValue: 123 }];
+
+ process.env.APIFY_DEFAULT_DATASET_ID = datasetId;
+
+ Apify.setPromisesDependency(Promise);
+
+ const mock = sinon.mock(Apify.client.datasets);
+ mock.expects('putItem')
+ .once()
+ .withArgs({
+ datasetId,
+ promise: Promise,
+ data: value,
+ })
+ .returns(Promise.resolve(null));
+
+ return Promise.resolve()
+ .then(() => {
+ return Apify.pushData(value);
})
.then(() => {
mock.verify();