Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
User-friendly JSON formatting in Apify.setValue(), fixed broken user agent tests, bumped package.json version
Jan Curndb2c5409e526
diff --git a/package.json b/package.jsonindex a2923daa3..d0b5a1647 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.4.7",+ "version": "0.4.8","description": "Apify runtime for JavaScript","main": "build/index.js","keywords": [diff --git a/src/actor.js b/src/actor.jsindex 34ea9311b..eb49d75d3 100644--- a/src/actor.js+++ b/src/actor.js@@ -122,7 +122,8 @@ export const setValue = (key, value, options, callback = null) => {if (options.contentType === null || options.contentType === undefined) {options.contentType = 'application/json';try {- value = JSON.stringify(value);+ // Format JSON to simplify debugging, the overheads with compression is negligible+ value = JSON.stringify(value, null, 2);} catch (e) {throw new Error(`The "value" parameter cannot be stringified to JSON: ${e.message}`);}diff --git a/test/actor.js b/test/actor.jsindex e6b5457ab..805199e1c 100644--- a/test/actor.js+++ b/test/actor.js@@ -611,7 +611,7 @@ describe('Apify.setValue()', () => {storeId,promise: Promise,key,- body: JSON.stringify(value),+ body: JSON.stringify(value, null, 2),contentType: 'application/json',useRawBody: true,})diff --git a/test/browser.js b/test/browser.jsindex 56ffe6551..36e7c0457 100644--- a/test/browser.js+++ b/test/browser.js@@ -275,7 +275,8 @@ describe('Apify.browse()', function () {it('userAgent option works', () => {let browser;const opts = {- url: 'http://www.whatsmyua.info/',+ // TODO: this is not reliable, we should use our own testing page+ url: 'http://www.whoishostingthis.com/tools/user-agent/',headless: true,browserName: 'chrome',userAgent: 'MyUserAgent/1234',@@ -289,12 +290,34 @@ describe('Apify.browse()', function () {return browser.webDriver.getPageSource();}).then((source) => {- expect(source).to.contain(`rawUa: ${opts.userAgent}`);+ expect(source).to.contain(opts.userAgent);return browser.close();});});- /*+/*+ it('test cache dir', () => {+ console.log('HEREE');+ let browser;+ const opts = {+ url: 'https://www.apifier.com/',+ headless: false,+ browserName: 'chrome',+ extraChromeArguments: ['--disk-cache-dir=/Users/jan/Projects/apify-runtime-js/_profiles/xxx1'], //, '--disk-cache-size=10000000'],+ };+ Apify.setPromisesDependency(Promise);+ return Apify.browse(opts)+ .then((result) => {+ browser = result;+ })+ .then(() => {+ // return browser.webDriver.sleep(300 * 1000);+ })+ .finally(() => {+ // return browser.close();+ });+ });+it('test xxx', () => {// const _ = require('underscore');// const Apify = require('apify');