Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Renamed sequential store to dataset

Jaroslav Hejlek765c5849a1af
diff --git a/src/actor.js b/src/actor.js
index 3da68a568..febd1b444 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -52,10 +52,10 @@ const getDefaultStoreIdOrThrow = () => {
return storeId;
};
-const getDefaultSequentialStoreIdOrThrow = () => {
- const storeId = process.env[ENV_VARS.DEFAULT_SEQUENTIAL_STORE_ID];
- if (!storeId) throw new Error(`The '${ENV_VARS.DEFAULT_SEQUENTIAL_STORE_ID}' environment variable is not defined.`);
- return storeId;
+const getDefaultDatasetIdOrThrow = () => {
+ const datasetId = process.env[ENV_VARS.DEFAULT_DATASET_ID];
+ if (!datasetId) throw new Error(`The '${ENV_VARS.DEFAULT_DATASET_ID}' environment variable is not defined.`);
+ return datasetId;
};
@@ -305,44 +305,43 @@ export const setValue = (key, value, options, callback = null) => {
* @ignore
* @memberof module:Apify
* @function
- * @description <p>Stores a record (object) in a sequential store using the Apify API.
- * If this is first write then a new store is created and associated with this act and then this and all further call
- * are stored in it. Default id of the store is in the `APIFY_DEFAULT_SEQUENTIAL_STORE_ID` environment variable;
+ * @description <p>Stores an item (object) in a dataset using the Apify API.
* The function has no result, but throws on invalid args or other errors.</p>
- * <pre><code class="language-javascript">await Apify.pushRecord(record);</code></pre>
+ * <pre><code class="language-javascript">await Apify.pushItem(item);</code></pre>
* <p>
* By default, the record is stored as is in default sequential store associated with this act.
+ * </p>
* <p>
- * **IMPORTANT: Do not forget to use the `await` keyword when calling `Apify.pushRecord()`,
+ * **IMPORTANT: Do not forget to use the `await` keyword when calling `Apify.pushItem()`,
* otherwise the act process might finish before the record is stored!**
* </p>
- * @param {Object} record Object containing date to by stored in the store
+ * @param {Object} item Object containing data to by stored in the dataset
* @param {Function} [callback] Optional callback. Function returns a promise if not provided.
* @returns {Promise} Returns a promise if `callback` was not provided.
*/
-export const pushRecord = (record, callback = null) => {
- if (!record || !_.isObject(record) || _.isArray(record)) throw new Error('The "record" parameter must be an object');
+export const pushItem = (item, callback = null) => {
+ if (!record || !_.isObject(record) || _.isArray(record)) throw new Error('The "item" parameter must be an object');
if (callback && !_.isFunction(callback)) throw new Error('If provided then the "callback" parameter must be a function');
const promisePrototype = getPromisePrototype();
- let stringifiedRecord;
+ let stringifiedItem;
try {
// Format JSON to simplify debugging, the overheads with compression is negligible
- stringifiedRecord = JSON.stringify(record, null, 2);
+ stringifiedItem = JSON.stringify(item, null, 2);
} catch (e) {
- throw new Error(`The "record" parameter cannot be stringified to JSON: ${e.message}`);
+ throw new Error(`The "item" parameter cannot be stringified to JSON: ${e.message}`);
}
- if (stringifiedRecord === undefined) {
- throw new Error('The "record" parameter cannot be stringified to JSON.');
+ if (stringifiedItem === undefined) {
+ throw new Error('The "item" parameter cannot be stringified to JSON.');
}
- const storeId = getDefaultSequentialStoreIdOrThrow();
+ const datasetId = getDefaultDatasetIdOrThrow();
- const innerPromise = apifyClient.sequentialStores.putRecord({
- storeId,
+ const innerPromise = apifyClient.datasets.putItem({
+ datasetId,
promise: promisePrototype,
- data: record,
+ data: item,
});
diff --git a/test/actor.js b/test/actor.js
index 991fac53b..4584b237d 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -939,50 +939,50 @@ describe('Apify.setValue()', () => {
});
});
-describe('Apify.pushRecord()', () => {
+describe('Apify.pushItem()', () => {
it('throws on invalid args', () => {
- process.env.APIFY_DEFAULT_SEQUENTIAL_STORE_ID = '1234';
- const recordErrMsg = 'The "record" parameter must be an object';
- expect(() => { Apify.pushRecord(); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushRecord(''); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushRecord(123); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushRecord(true); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushRecord(false); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushRecord([]); }).to.throw(Error, recordErrMsg);
- expect(() => { Apify.pushRecord(() => {}, () => {}); }).to.throw(Error, jsonErrMsg);
+ process.env.APIFY_DEFAULT_DATASET_ID = '1234';
+ const recordErrMsg = 'The "item" parameter must be an object';
+ expect(() => { Apify.pushItem(); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushItem(''); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushItem(123); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushItem(true); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushItem(false); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushItem([]); }).to.throw(Error, recordErrMsg);
+ expect(() => { Apify.pushItem(() => {}, () => {}); }).to.throw(Error, jsonErrMsg);
const circularObj = {};
circularObj.xxx = circularObj;
- const jsonErrMsg = 'The "record" parameter cannot be stringified to JSON';
- expect(() => { Apify.pushRecord(circularObj, null); }).to.throw(Error, jsonErrMsg);
+ const jsonErrMsg = 'The "item" parameter cannot be stringified to JSON';
+ expect(() => { Apify.pushItem(circularObj, null); }).to.throw(Error, jsonErrMsg);
});
- it('throws if APIFY_DEFAULT_SEQUENTIAL_STORE_ID env var is not defined', () => {
- const errMsg = 'The \'APIFY_DEFAULT_SEQUENTIAL_STORE_ID\' environment variable is not defined';
+ it('throws if APIFY_DEFAULT_DATASET_ID env var is not defined', () => {
+ const errMsg = 'The \'APIFY_DEFAULT_DATASET_ID\' environment variable is not defined';
- process.env.APIFY_DEFAULT_SEQUENTIAL_STORE_ID = '';
- expect(() => { Apify.pushRecord({ something: 123 }); }).to.throw(Error, errMsg);
+ process.env.APIFY_DEFAULT_DATASET_ID = '';
+ expect(() => { Apify.pushItem({ something: 123 }); }).to.throw(Error, errMsg);
delete process.env.APIFY_DEFAULT_SEQUENTIAL_STORE_ID;
- expect(() => { Apify.pushRecord({ something: 123 }); }).to.throw(Error, errMsg);
+ expect(() => { Apify.pushItem({ something: 123 }); }).to.throw(Error, errMsg);
});
it('supports both promises and callbacks (on success)', () => {
- process.env.APIFY_DEFAULT_SEQUENTIAL_STORE_ID = '1234';
- const mock = sinon.mock(Apify.client.sequentialStores);
- mock.expects('putRecord')
+ process.env.APIFY_DEFAULT_DATASET_ID = '1234';
+ const mock = sinon.mock(Apify.client.datasets);
+ mock.expects('putItem')
.twice()
.returns(Promise.resolve(null));
return Promise.resolve()
.then(() => {
// test promise (no options)
- return Apify.pushRecord({ someValue: 123 });
+ return Apify.pushItem({ someValue: 123 });
})
.then(() => {
// test callback (no options)
return new Promise((resolve, reject) => {
- Apify.pushRecord({ someValue: 123 }, (err) => {
+ Apify.pushItem({ someValue: 123 }, (err) => {
if (err) return reject(err);
resolve();
});
@@ -997,9 +997,9 @@ describe('Apify.pushRecord()', () => {
});
it('supports both promises and callbacks (on error)', () => {
- process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
- const mock = sinon.mock(Apify.client.sequentialStores);
- mock.expects('putRecord')
+ process.env.APIFY_DEFAULT_DATASET_ID = '1234';
+ const mock = sinon.mock(Apify.client.datasets);
+ mock.expects('putItem')
.twice()
.throws(new Error('Test error'));
@@ -1008,7 +1008,7 @@ describe('Apify.pushRecord()', () => {
// Test promise
return Promise.resolve()
.then(() => {
- return Apify.pushRecord({ someValue: 1 });
+ return Apify.pushItem({ someValue: 1 });
})
.then(() => {
assert.fail();
@@ -1020,7 +1020,7 @@ describe('Apify.pushRecord()', () => {
.then(() => {
// Test callback
return new Promise((resolve, reject) => {
- Apify.pushRecord({ someValue: 1 }, (err) => {
+ Apify.pushItem({ someValue: 1 }, (err) => {
if (err) return reject(err);
resolve();
});
@@ -1041,18 +1041,18 @@ describe('Apify.pushRecord()', () => {
});
it('correctly stores records', () => {
- const storeId = 'mystore';
+ const datasetId = 'mystore';
const value = { someValue: 123 };
- process.env.APIFY_DEFAULT_SEQUENTIAL_STORE_ID = storeId;
+ process.env.APIFY_DEFAULT_DATASET_ID = datasetId;
Apify.setPromisesDependency(Promise);
const mock = sinon.mock(Apify.client.sequentialStores);
- mock.expects('putRecord')
+ mock.expects('putItem')
.once()
.withArgs({
- storeId,
+ datasetId,
promise: Promise,
data: value,
})
@@ -1060,7 +1060,7 @@ describe('Apify.pushRecord()', () => {
return Promise.resolve()
.then(() => {
- return Apify.pushRecord(value);
+ return Apify.pushItem(value);
})
.then(() => {
mock.verify();