Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Progress commit

Ondra Urban74d10c8fc74f
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.js
index 6b097a8a5..8ee3390ef 100644
--- a/src/live_view/live_view_server.js
+++ b/src/live_view/live_view_server.js
@@ -2,10 +2,11 @@ const http = require('http');
const fs = require('fs');
const express = require('express');
const socketio = require('socket.io');
-const utils = require('apify-shared/utilities');
-const log = require('apify-shared/log');
+const { promisifyServerListen } = require('apify-shared/utilities');
const { ENV_VARS, LOCAL_ENV_VARS } = require('apify-shared/consts');
+const { utils: { log } } = Apify;
+
const DELETE_SCREENSHOT_FILE_TIMEOUT_MILLIS = 10 * 1000;
class LiveViewServer {
@@ -14,8 +15,8 @@ class LiveViewServer {
this.port = parseInt(containerPort, 10);
if (!(this.port >= 0 && this.port <= 65535)) {
- throw new Error(`Cannot start LiveViewServer - invalid port specified by the ${
- ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);
+ throw new Error('Cannot start LiveViewServer - invalid port specified by the '
+ + `${ENV_VARS.CONTAINER_PORT} environment variable (was "${containerPort}").`);
}
this.liveViewUrl = process.env[ENV_VARS.CONTAINER_URL] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_URL];
@@ -71,7 +72,7 @@ class LiveViewServer {
* Starts the HTTP server.
*/
async start() {
- await utils.promisifyServerListen(this.httpServer)(this.port);
+ await promisifyServerListen(this.httpServer)(this.port);
log.info('Live view web server started', { publicUrl: this.liveViewUrl });
}
@@ -87,7 +88,7 @@ class LiveViewServer {
this.lastSnapshot = {
pageUrl,
htmlContent,
- screenshotIndex: this.lastScreenshotIndex
+ screenshotIndex: this.lastScreenshotIndex,
};
this.screenshotIndexToFilePath[this.lastScreenshotIndex] = screenshotFilePath;
@@ -106,7 +107,7 @@ class LiveViewServer {
});
}, DELETE_SCREENSHOT_FILE_TIMEOUT_MILLIS);
}
- };
+ }
hasClients() {
return this.clientCount > 0;
diff --git a/src/puppeteer.js b/src/puppeteer.js
index e352c47dd..82ff7bfae 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -4,7 +4,7 @@ import { anonymizeProxy, closeAnonymizedProxy, redactUrl } from 'proxy-chain';
import log from 'apify-shared/log';
import { ENV_VARS } from 'apify-shared/consts';
import { DEFAULT_USER_AGENT } from './constants';
-import { newPromise, getTypicalChromeExecutablePath, isAtHome } from './utils';
+import { getTypicalChromeExecutablePath, isAtHome } from './utils';
import { getApifyProxyUrl } from './actor';
import { registerBrowserForLiveView } from './puppeteer_live_view_server';
@@ -62,6 +62,9 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
* will use the same target proxy server (i.e. the same IP address).
* The identifier can only contain the following characters: `0-9`, `a-z`, `A-Z`, `"."`, `"_"` and `"~"`.
* Only applied if the `useApifyProxy` option is `true`.
+ * @property ${String} [puppeteerModule]
+ * Require path to a module to be used instead of default 'puppeteer'. This enables usage
+ * of various Puppeteer wrappers such as 'puppeteer-extra'.
* @property {Boolean} [liveView=false]
* If set to `true`, a `PuppeteerLiveViewServer` will be started to enable
* screenshot and html capturing of visited pages using `PuppeteerLiveViewBrowser`.
@@ -80,60 +83,52 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
*
* @ignore
*/
-const launchPuppeteerWithProxy = (puppeteer, opts) => {
- let anonymizedProxyUrl;
-
+const launchPuppeteerWithProxy = async (puppeteer, opts) => {
// Parse and validate proxy URL and anonymize it
- // NOTE: anonymizeProxy() throws on invalid proxy URL, so it must not be inside a Promise!
- return anonymizeProxy(opts.proxyUrl)
- .then((result) => {
- anonymizedProxyUrl = result;
- opts.args.push(`--proxy-server=${anonymizedProxyUrl}`);
- })
- .then(() => {
- const optsForLog = _.omit(opts, LAUNCH_PUPPETEER_LOG_OMIT_OPTS);
- optsForLog.proxyUrl = redactUrl(opts.proxyUrl);
- optsForLog.args = opts.args.slice(0, opts.args.length - 1);
- log.info('Launching Puppeteer', optsForLog);
+ const anonymizedProxyUrl = await anonymizeProxy(opts.proxyUrl);
+ opts.args.push(`--proxy-server=${anonymizedProxyUrl}`);
+ const optsForLog = _.omit(opts, LAUNCH_PUPPETEER_LOG_OMIT_OPTS);
+ optsForLog.proxyUrl = redactUrl(opts.proxyUrl);
+ optsForLog.args = opts.args.slice(0, opts.args.length - 1);
- return puppeteer.launch(opts);
- })
- // Close anonymization proxy server when Puppeteer finishes
- .then((browser) => {
- const cleanUp = () => {
- // Don't wait for finish, only log errors
- closeAnonymizedProxy(anonymizedProxyUrl, true)
- .catch((err) => {
- console.log(`WARNING: closeAnonymizedProxy() failed with: ${err.stack || err}`);
- });
- };
+ log.info('Launching Puppeteer', optsForLog);
+ const browser = await puppeteer.launch(opts);
- browser.on('disconnected', cleanUp);
+ // Close anonymization proxy server when Puppeteer finishes
+ const cleanUp = () => {
+ // Don't wait for finish, only log errors
+ closeAnonymizedProxy(anonymizedProxyUrl, true)
+ .catch(err => log.exception(err, 'closeAnonymizedProxy() failed.'));
+ };
- const prevClose = browser.close.bind(browser);
- browser.close = () => {
- cleanUp();
- return prevClose();
- };
+ browser.on('disconnected', cleanUp);
- return browser;
- });
+ const prevClose = browser.close.bind(browser);
+ browser.close = () => {
+ cleanUp();
+ return prevClose();
+ };
+
+ return browser;
};
/**
* Requires `puppeteer` package or throws meaningful error if not installed.
*
+ * @param {string} puppeteerModule
* @ignore
*/
-const getPuppeteerOrThrow = () => {
+const getPuppeteerOrThrow = (puppeteerModule = 'puppeteer') => {
+ checkParamOrThrow(puppeteerModule, 'puppeteerModule', 'String');
try {
// This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)
- return require('puppeteer'); // eslint-disable-line global-require
+ return require(puppeteerModule); // eslint-disable-line
} catch (err) {
if (err.code === 'MODULE_NOT_FOUND') {
+ const msg = `Cannot find module '${puppeteerModule}'. Did you you install the '${puppeteerModule}' package?`;
err.message = isAtHome()
- ? 'Cannot find module \'puppeteer\'. Did you choose the correct base Docker image (apify/actor-node-chrome-*)?'
- : 'Cannot find module \'puppeteer\'. Did you you install \'puppeteer\' package?';
+ ? `${msg} The 'puppeteer' package is automatically bundled when using apify/actor-node-chrome-* Base image.`
+ : msg;
}
throw err;
@@ -200,16 +195,17 @@ const getPuppeteerOrThrow = () => {
* @name launchPuppeteer
* @function
*/
-export const launchPuppeteer = (options = {}) => {
+export const launchPuppeteer = async (options = {}) => {
checkParamOrThrow(options, 'options', 'Object');
checkParamOrThrow(options.args, 'options.args', 'Maybe [String]');
checkParamOrThrow(options.proxyUrl, 'options.proxyUrl', 'Maybe String');
checkParamOrThrow(options.useApifyProxy, 'options.useApifyProxy', 'Maybe Boolean');
+ checkParamOrThrow(options.puppeteerModule, 'options.puppeteerModule', 'Maybe String');
checkParamOrThrow(options.liveView, 'options.liveView', 'Maybe Boolean');
checkParamOrThrow(options.liveViewOptions, 'options.liveViewOptions', 'Maybe Object');
if (options.useApifyProxy && options.proxyUrl) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrl"!');
- const puppeteer = getPuppeteerOrThrow();
+ const puppeteer = getPuppeteerOrThrow(options.puppeteerModule);
const optsCopy = Object.assign({}, options);
optsCopy.args = optsCopy.args || [];
@@ -253,11 +249,11 @@ export const launchPuppeteer = (options = {}) => {
browserPromise = puppeteer.launch(optsCopy);
}
- // Ensure that the returned promise is of type Bluebird.
- const wrapped = newPromise().then(() => browserPromise);
-
// Start LiveView server if requested
- if (optsCopy.liveView) return registerBrowserForLiveView(wrapped, optsCopy.liveViewOptions).then(() => wrapped);
+ if (optsCopy.liveView) {
+ const browser = await browserPromise;
+ return registerBrowserForLiveView(browser, optsCopy.liveViewOptions).then(() => wrapped);
+ }
return wrapped;
};