Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

User-friendly JSON formatting in Apify.setValue(), fixed broken user agent tests, bumped package.json version

Jan Curndb2c5409e526
diff --git a/package.json b/package.json
index a2923daa3..d0b5a1647 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "0.4.7",
+ "version": "0.4.8",
"description": "Apify runtime for JavaScript",
"main": "build/index.js",
"keywords": [
diff --git a/src/actor.js b/src/actor.js
index 34ea9311b..eb49d75d3 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -122,7 +122,8 @@ export const setValue = (key, value, options, callback = null) => {
if (options.contentType === null || options.contentType === undefined) {
options.contentType = 'application/json';
try {
- value = JSON.stringify(value);
+ // Format JSON to simplify debugging, the overheads with compression is negligible
+ value = JSON.stringify(value, null, 2);
} catch (e) {
throw new Error(`The "value" parameter cannot be stringified to JSON: ${e.message}`);
}
diff --git a/test/actor.js b/test/actor.js
index e6b5457ab..805199e1c 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -611,7 +611,7 @@ describe('Apify.setValue()', () => {
storeId,
promise: Promise,
key,
- body: JSON.stringify(value),
+ body: JSON.stringify(value, null, 2),
contentType: 'application/json',
useRawBody: true,
})
diff --git a/test/browser.js b/test/browser.js
index 56ffe6551..36e7c0457 100644
--- a/test/browser.js
+++ b/test/browser.js
@@ -275,7 +275,8 @@ describe('Apify.browse()', function () {
it('userAgent option works', () => {
let browser;
const opts = {
- url: 'http://www.whatsmyua.info/',
+ // TODO: this is not reliable, we should use our own testing page
+ url: 'http://www.whoishostingthis.com/tools/user-agent/',
headless: true,
browserName: 'chrome',
userAgent: 'MyUserAgent/1234',
@@ -289,12 +290,34 @@ describe('Apify.browse()', function () {
return browser.webDriver.getPageSource();
})
.then((source) => {
- expect(source).to.contain(`rawUa: ${opts.userAgent}`);
+ expect(source).to.contain(opts.userAgent);
return browser.close();
});
});
- /*
+/*
+ it('test cache dir', () => {
+ console.log('HEREE');
+ let browser;
+ const opts = {
+ url: 'https://www.apifier.com/',
+ headless: false,
+ browserName: 'chrome',
+ extraChromeArguments: ['--disk-cache-dir=/Users/jan/Projects/apify-runtime-js/_profiles/xxx1'], //, '--disk-cache-size=10000000'],
+ };
+ Apify.setPromisesDependency(Promise);
+ return Apify.browse(opts)
+ .then((result) => {
+ browser = result;
+ })
+ .then(() => {
+ // return browser.webDriver.sleep(300 * 1000);
+ })
+ .finally(() => {
+ // return browser.close();
+ });
+ });
+
it('test xxx', () => {
// const _ = require('underscore');
// const Apify = require('apify');