Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Changed Apify.setValue() params - replaced contentType with options, updated unit tests and README, bumped package.json version

Jan Curnc308cf1e0fed
diff --git a/README.md b/README.md
index f03dedbdc..685acad2b 100644
--- a/README.md
+++ b/README.md
@@ -142,8 +142,9 @@ console.log('My input:');
console.dir(input);
```
-If the input data has the `application/json` content type, it is automatically parsed into a JavaScript object,
-otherwise it is left as a String or Buffer in the raw form.
+If the input data has the `application/json` content type, it is automatically parsed into a JavaScript object.
+For the `text/plain` content type the result is a string.
+For other content types, the result is raw Buffer.
Similarly, the output can be stored as follows:
@@ -154,7 +155,18 @@ const output = {
await Apify.setValue('OUTPUT', output);
```
-**IMPORTANT: Do not forget to use the `await` keyword, otherwise the act's process might finish before the output is stored!**
+By default, the value is converted to JSON and stored with the `application/json` content type.
+If you want to store the data with another content type, pass it in the options as follows:
+
+```javascript
+await Apify.setValue('OUTPUT', 'my text data', { contentType: 'text/plain' });
+```
+
+**IMPORTANT: Do not forget to use the `await` keyword when calling `Apify.setValue()`,
+otherwise the act's process might finish before the output is stored and/or storage errors will be ignored!**
+
+Besides the key `INPUT` and `OUTPUT`, you can use arbitrary keys
+to store any data from your act, such as its state or larger results.
### Browser
diff --git a/package.json b/package.json
index b34bebf61..22a65d3f7 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "0.4.4",
+ "version": "0.4.5",
"description": "Apify runtime for JavaScript",
"main": "build/index.js",
"keywords": [
diff --git a/src/actor.js b/src/actor.js
index e5a5487f2..3ba2b1cff 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -92,22 +92,26 @@ export const getValue = (key, callback = null) => {
* The function has no result, but throws on invalid args or other errors.
* @param value
* If null, the record in the key-value store is deleted.
- * If no contentType, the value can be any object and it will be stringified to JSON.
+ * If no contentType is specified, the value can be any object and it will be stringified to JSON.
* If contentType is specified, value is considered raw data it must be a String or Buffer.
* For any other value an error will be thrown.
- * @param contentType Optional MIME content type of the value.
+ * @param options Optional settings, currently only { contentType: String } is supported to set MIME content type of the value.
* @param callback Optional callback.
* @return Returns a promise if no callback was provided, otherwise the return value is not defined.
*/
-export const setValue = (key, value, contentType, callback = null) => {
+export const setValue = (key, value, options, callback = null) => {
if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');
// contentType is optional
- if (_.isFunction(contentType)) {
- callback = contentType;
- contentType = null;
+ if (_.isFunction(options)) {
+ callback = options;
+ options = null;
}
+ if (typeof (options) !== 'object' && options !== undefined) throw new Error('The "options" parameter must be an object, null or undefined.');
+ // Make copy of options, don't update what user passed
+ options = Object.assign({}, options);
+
const storeId = getDefaultStoreIdOrThrow();
const promisePrototype = getPromisePrototype();
@@ -115,8 +119,8 @@ export const setValue = (key, value, contentType, callback = null) => {
if (value !== null) {
// Normal case: put record to store
// If contentType is missing, value will be stringified to JSON
- if (contentType === null || contentType === undefined) {
- contentType = 'application/json';
+ if (options.contentType === null || options.contentType === undefined) {
+ options.contentType = 'application/json';
try {
value = JSON.stringify(value);
} catch (e) {
@@ -127,8 +131,8 @@ export const setValue = (key, value, contentType, callback = null) => {
}
}
- if (!contentType || !_.isString(contentType)) {
- throw new Error('The "contentType" parameter must be a non-empty string, null or undefined.');
+ if (!options.contentType || !_.isString(options.contentType)) {
+ throw new Error('The "options.contentType" parameter must be a non-empty string, null or undefined.');
}
if (!_.isString(value) && !Buffer.isBuffer(value)) {
throw new Error('The "value" parameter must be a String or Buffer when "contentType" is specified.');
@@ -140,13 +144,13 @@ export const setValue = (key, value, contentType, callback = null) => {
promise: promisePrototype,
key,
body: value,
- contentType,
+ contentType: options.contentType,
useRawBody: true,
});
} else {
// Special case: remove the record from the store
- if (contentType !== null && contentType !== undefined) {
- throw new Error('The "contentType" parameter must not be used when removing the record.');
+ if (options.contentType !== null && options.contentType !== undefined) {
+ throw new Error('The "options.contentType" parameter must not be used when removing the record.');
}
innerPromise = apifyClient.keyValueStores.deleteRecord({
storeId,
diff --git a/test/actor.js b/test/actor.js
index aa2d924e0..e6b5457ab 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -452,9 +452,14 @@ describe('Apify.setValue()', () => {
expect(() => { Apify.setValue(123, 'some value'); }).to.throw(Error, keyErrMsg);
const valueErrMsg = 'The "value" parameter must be a String or Buffer when "contentType" is specified';
- expect(() => { Apify.setValue('key', {}, 'image/png'); }).to.throw(Error, valueErrMsg);
- expect(() => { Apify.setValue('key', 12345, 'image/png'); }).to.throw(Error, valueErrMsg);
- expect(() => { Apify.setValue('key', () => {}, 'image/png'); }).to.throw(Error, valueErrMsg);
+ expect(() => { Apify.setValue('key', {}, { contentType: 'image/png' }); }).to.throw(Error, valueErrMsg);
+ expect(() => { Apify.setValue('key', 12345, { contentType: 'image/png' }); }).to.throw(Error, valueErrMsg);
+ expect(() => { Apify.setValue('key', () => {}, { contentType: 'image/png' }); }).to.throw(Error, valueErrMsg);
+
+ const optsErrMsg = 'The "options" parameter must be an object, null or undefined';
+ expect(() => { Apify.setValue('key', {}, 123); }).to.throw(Error, optsErrMsg);
+ expect(() => { Apify.setValue('key', {}, 'bla/bla'); }).to.throw(Error, optsErrMsg);
+ expect(() => { Apify.setValue('key', {}, true); }).to.throw(Error, optsErrMsg);
const circularObj = {};
circularObj.xxx = circularObj;
@@ -464,16 +469,16 @@ describe('Apify.setValue()', () => {
expect(() => { Apify.setValue('key', () => {}); }).to.throw(Error, jsonErrMsg);
expect(() => { Apify.setValue('key'); }).to.throw(Error, jsonErrMsg);
- const contTypeRedundantErrMsg = 'The "contentType" parameter must not be used when removing the record';
- expect(() => { Apify.setValue('key', null, 'image/png'); }).to.throw(Error, contTypeRedundantErrMsg);
- expect(() => { Apify.setValue('key', null, ''); }).to.throw(Error, contTypeRedundantErrMsg);
- expect(() => { Apify.setValue('key', null, {}); }).to.throw(Error, contTypeRedundantErrMsg);
+ const contTypeRedundantErrMsg = 'The "options.contentType" parameter must not be used when removing the record';
+ expect(() => { Apify.setValue('key', null, { contentType: 'image/png' }); }).to.throw(Error, contTypeRedundantErrMsg);
+ expect(() => { Apify.setValue('key', null, { contentType: '' }); }).to.throw(Error, contTypeRedundantErrMsg);
+ expect(() => { Apify.setValue('key', null, { contentType: {} }); }).to.throw(Error, contTypeRedundantErrMsg);
- const contTypeStringErrMsg = 'The "contentType" parameter must be a non-empty string, null or undefined';
- expect(() => { Apify.setValue('key', 'value', 123); }).to.throw(Error, contTypeStringErrMsg);
- expect(() => { Apify.setValue('key', 'value', {}); }).to.throw(Error, contTypeStringErrMsg);
- expect(() => { Apify.setValue('key', 'value', new Date()); }).to.throw(Error, contTypeStringErrMsg);
- expect(() => { Apify.setValue('key', 'value', ''); }).to.throw(Error, contTypeStringErrMsg);
+ const contTypeStringErrMsg = 'The "options.contentType" parameter must be a non-empty string, null or undefined';
+ expect(() => { Apify.setValue('key', 'value', { contentType: 123 }); }).to.throw(Error, contTypeStringErrMsg);
+ expect(() => { Apify.setValue('key', 'value', { contentType: {} }); }).to.throw(Error, contTypeStringErrMsg);
+ expect(() => { Apify.setValue('key', 'value', { contentType: new Date() }); }).to.throw(Error, contTypeStringErrMsg);
+ expect(() => { Apify.setValue('key', 'value', { contentType: '' }); }).to.throw(Error, contTypeStringErrMsg);
});
it('throws if APIFY_DEFAULT_KEY_VALUE_STORE_ID env var is not defined', () => {
@@ -490,42 +495,68 @@ describe('Apify.setValue()', () => {
process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = '1234';
const mock = sinon.mock(Apify.client.keyValueStores);
mock.expects('putRecord')
- .exactly(4)
+ .exactly(8)
.returns(Promise.resolve(null));
return Promise.resolve()
- .then(() => {
- // test promise (no content type)
- return Apify.setValue('mykey', { someValue: 123 });
- })
- .then(() => {
- // test promise (with content type)
- return Apify.setValue('mykey', 'value', 'text/plain');
- })
- .then(() => {
- // test callback (no content type)
- return new Promise((resolve, reject) => {
- return Apify.setValue('mykey', { someValue: 123 }, (err) => {
- if (err) return reject(err);
- resolve();
+ .then(() => {
+ // test promise (no options)
+ return Apify.setValue('mykey', { someValue: 123 });
+ })
+ .then(() => {
+ // test promise (with options)
+ return Apify.setValue('mykey', 'value', { contentType: 'text/plain' });
+ })
+ .then(() => {
+ // test promise (null options)
+ return Apify.setValue('mykey', 'value', null);
+ })
+ .then(() => {
+ // test promise (undefined options)
+ return Apify.setValue('mykey', 'value', undefined);
+ })
+ .then(() => {
+ // test callback (no options)
+ return new Promise((resolve, reject) => {
+ return Apify.setValue('mykey', { someValue: 123 }, (err) => {
+ if (err) return reject(err);
+ resolve();
+ });
});
- });
- })
- .then(() => {
- // test callback (with content type)
- return new Promise((resolve, reject) => {
- return Apify.setValue('mykey', 'myvalue', 'text/plain', (err) => {
- if (err) return reject(err);
- resolve();
+ })
+ .then(() => {
+ // test callback (with options)
+ return new Promise((resolve, reject) => {
+ return Apify.setValue('mykey', 'myvalue', { contentType: 'text/plain' }, (err) => {
+ if (err) return reject(err);
+ resolve();
+ });
+ });
+ })
+ .then(() => {
+ // test callback (null options)
+ return new Promise((resolve, reject) => {
+ return Apify.setValue('mykey', { someValue: 123 }, null, (err) => {
+ if (err) return reject(err);
+ resolve();
+ });
+ });
+ })
+ .then(() => {
+ // test callback (undefined options)
+ return new Promise((resolve, reject) => {
+ return Apify.setValue('mykey', { someValue: 123 }, resolve, (err) => {
+ if (err) return reject(err);
+ resolve();
+ });
});
+ })
+ .then(() => {
+ mock.verify();
+ })
+ .finally(() => {
+ mock.restore();
});
- })
- .then(() => {
- mock.verify();
- })
- .finally(() => {
- mock.restore();
- });
});
it('supports both promises and callbacks (on error)', () => {
@@ -536,31 +567,31 @@ describe('Apify.setValue()', () => {
.throws(new Error('Test error'));
return Promise.resolve()
- .then(() => {
- // test promise
- return Apify.setValue('mykey', { someValue: 1 });
- })
- .catch((err) => {
- expect(err.message).to.be.eql('Test error');
- })
- .then(() => {
- // test callback
- return new Promise((resolve, reject) => {
- return Apify.setValue('mykey', { someValue: 1 }, (err) => {
- if (err) return reject(err);
- resolve();
+ .then(() => {
+ // test promise
+ return Apify.setValue('mykey', { someValue: 1 });
+ })
+ .catch((err) => {
+ expect(err.message).to.be.eql('Test error');
+ })
+ .then(() => {
+ // test callback
+ return new Promise((resolve, reject) => {
+ return Apify.setValue('mykey', { someValue: 1 }, (err) => {
+ if (err) return reject(err);
+ resolve();
+ });
});
+ })
+ .catch((err) => {
+ expect(err.message).to.be.eql('Test error');
+ })
+ .then(() => {
+ mock.verify();
+ })
+ .finally(() => {
+ mock.restore();
});
- })
- .catch((err) => {
- expect(err.message).to.be.eql('Test error');
- })
- .then(() => {
- mock.verify();
- })
- .finally(() => {
- mock.restore();
- });
});
it('correctly stores object values as JSON', () => {
@@ -624,7 +655,7 @@ describe('Apify.setValue()', () => {
return Promise.resolve()
.then(() => {
- return Apify.setValue(key, value, contentType);
+ return Apify.setValue(key, value, { contentType });
})
.then(() => {
mock.verify();
@@ -660,7 +691,7 @@ describe('Apify.setValue()', () => {
return Promise.resolve()
.then(() => {
- return Apify.setValue(key, value, contentType);
+ return Apify.setValue(key, value, { contentType });
})
.then(() => {
mock.verify();