Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/dev shm (#43)
Jan Čurn39296155ab72
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex bc2602875..13332509f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,9 @@+0.5.14 / 2018-03-20+===================+- Removed again the --disable-dev-shm-usage flag when launching headless Chrome,+ it might be causing issues with high IO overheads+- Upgraded Puppeteer to version 1.2.0+0.5.11 / 2018-03-09===================- Added back --no-sandbox to launch of Puppeteer to avoid issues on older kernelsdiff --git a/README.md b/README.mdindex 946936803..3ae42239b 100644--- a/README.md+++ b/README.md@@ -4,7 +4,9 @@[](http://badge.fury.io/js/apify)[](https://travis-ci.org/apifytech/apify-js)-The apify NPM package simplifies development of acts in <a href="https://www.apify.com/docs/actor" target="_blank">Apify Actor</a>++The `apify` NPM package enables development of web scrapers and crawlers,+either locally or running on <a href="https://www.apify.com/docs/actor" target="_blank">Apify Actor</a>- a serverless computing platform that enables execution of arbitrary pieces of code in the cloud. The package provides helper functions to launch web browsers with proxies, access the storage etc. Note that the usage of the package is optional, you can create acts without it.Complete documentation of this package is available at https://www.apify.com/docs/sdk/apify-runtime-js/latestdiff --git a/package.json b/package.jsonindex 577f88f67..d9cc5db9c 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.5.13",+ "version": "0.5.14","description": "Apify SDK for JavaScript / Node.js","main": "build/index.js","keywords": [@@ -85,7 +85,7 @@"why-is-node-running": "^1.2.3"},"optionalDependencies": {- "puppeteer": "^1.1.1",+ "puppeteer": "^1.2.0","selenium-webdriver": "^3.6.0"}}diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 1c0f52d2d..5c96ef562 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -22,8 +22,7 @@ import { newPromise } from './utils';* to make the proxy work with headless Chrome. For more information, read the* <a href="https://blog.apify.com/249a21a79212" target="_blank">blog post about proxy-chain library</a>.* </li>- * <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Apify Actor platform- * and `--disable-dev-shm-usage` to avoid using shared memory space in favour of `/tmp`.</li>+ * <li>Adds `--no-sandbox` to `args` to enable running headless Chrome in a Docker container on the Apify Actor platform.</li>* </ul>** To use this function, you need to have the <a href="https://www.npmjs.com/package/puppeteer" target="_blank">puppeteer</a>@@ -66,7 +65,6 @@ export const launchPuppeteer = (opts) => {opts.args = opts.args || [];opts.args.push('--no-sandbox');- opts.args.push('--disable-dev-shm-usage');opts.args.push(`--user-agent=${opts.userAgent || DEFAULT_USER_AGENT}`);if (opts.headless === undefined || opts.headless === null) {opts.headless = process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';diff --git a/src/webdriver.js b/src/webdriver.jsindex 2ccae0296..0c493df8a 100644--- a/src/webdriver.js+++ b/src/webdriver.js@@ -63,7 +63,7 @@ export class Browser {this.chromeOptions.addArguments('--disable-translate');this.chromeOptions.addArguments('--safebrowsing-disable-auto-update');if (this.options.headless) {- this.chromeOptions.addArguments('--headless', '--no-sandbox', '--disable-dev-shm-usage');+ this.chromeOptions.addArguments('--headless', '--no-sandbox');if (isWin) {// Temporarily needed on Windowsthis.chromeOptions.addArguments('--disable-gpu');diff --git a/test/puppeteer.js b/test/puppeteer.jsindex 2f4a4b8a6..e71f44ff9 100644--- a/test/puppeteer.js+++ b/test/puppeteer.js@@ -129,4 +129,32 @@ describe('Apify.launchPuppeteer()', () => {.then(html => expect(html).to.include('<h1>Example Domain</h1>')).then(() => browser.close());});++ it('userAgent option works', () => {+ let browser;+ let page;+ const opts = {+ // Have space in user-agent to test passing of params+ userAgent: 'MyUserAgent/1234 AnotherString/456',+ headless: true,+ };+ return Apify.launchPuppeteer(opts)+ .then((result) => {+ browser = result;+ })+ .then(() => {+ return browser.newPage();+ })+ .then((result) => {+ page = result;+ return page.goto('https://api.apify.com/v2/browser-info');+ })+ .then(() => {+ return page.content();+ })+ .then((html) => {+ expect(html).to.contain(`"user-agent": "${opts.userAgent}"`);+ return browser.close();+ });+ });});diff --git a/test/webdriver.js b/test/webdriver.jsindex 952c89b1e..91b131f39 100644--- a/test/webdriver.js+++ b/test/webdriver.js@@ -217,15 +217,14 @@ describe('Apify.launchWebDriver()', function () {return Apify.launchWebDriver(opts).then((result) => {webDriver = result;- // TODO: this is not reliable, we should use our own testing page- return webDriver.get('http://www.whoishostingthis.com/tools/user-agent');+ return webDriver.get('https://api.apify.com/v2/browser-info');}).then(() => {expect(webDriver.constructor.name).to.eql('Driver');return webDriver.getPageSource();}).then((source) => {- expect(source).to.contain(opts.userAgent);+ expect(source).to.contain(`"user-agent": "${opts.userAgent}"`);return webDriver.quit();});});@@ -311,8 +310,7 @@ describe('Apify.browse()', function () {it('userAgent option works', () => {let browser;const opts = {- // TODO: this is not reliable, we should use our own testing page- url: 'http://www.whoishostingthis.com/tools/user-agent/',+ url: 'https://api.apify.com/v2/browser-info',headless: true,browserName: 'chrome',// Have space in user-agent to test passing of params@@ -327,7 +325,7 @@ describe('Apify.browse()', function () {return browser.webDriver.getPageSource();}).then((source) => {- expect(source).to.contain(opts.userAgent);+ expect(source).to.contain(`"user-agent": "${opts.userAgent}"`);return browser.close();});});