Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Created browser launchers (#900)

petrpatekb313fcad8073
diff --git a/src/browser_launchers/browser_launcher.js b/src/browser_launchers/browser_launcher.js
new file mode 100644
index 000000000..47be72753
--- /dev/null
+++ b/src/browser_launchers/browser_launcher.js
@@ -0,0 +1,163 @@
+import ow from 'ow';
+import { ENV_VARS } from 'apify-shared/consts';
+import { getTypicalChromeExecutablePath, isAtHome } from '../utils';
+
+/**
+ * @typedef BrowserLaunchContext
+ * @property {object} [launchOptions]
+ * `Options passed to the browser launcher function. Options are based on underlying library.
+ * @property {string} [proxyUrl]
+ * URL to a HTTP proxy server. It must define the port number,
+ * and it may also contain proxy username and password.
+ *
+ * Example: `http://bob:[email protected]:1234`.
+ * @property {boolean} [useChrome=false]
+ * If `true` and `executablePath` is not set,
+ * Playwright will launch full Google Chrome browser available on the machine
+ * rather than the bundled Chromium. The path to Chrome executable
+ * is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
+ * or defaults to the typical Google Chrome executable location specific for the operating system.
+ * By default, this option is `false`.
+ * @property {Object} [launcher]
+ * By default this function uses require("playwright").chromium`.
+ * If you want to use a different browser you can pass it by this property as `require("playwright").firefox
+ */
+
+/**
+ * Abstract class for creating browser launchers, such as `PlaywrightLauncher` and `PuppeteerLauncher`.
+ */
+export default class BrowserLauncher {
+ static optionsShape = {
+ launcher: ow.object,
+ proxyUrl: ow.optional.string.url,
+ useChrome: ow.optional.boolean,
+ launchOptions: ow.optional.object,
+ }
+
+ /**
+ *
+ * @param {string} launcher
+ * @param {string} apifyImageName
+ * @returns {object}
+ *
+ */
+ static requireLauncherOrThrow(launcher, apifyImageName) {
+ try {
+ return require(launcher); // eslint-disable-line
+ } catch (err) {
+ if (err.code === 'MODULE_NOT_FOUND') {
+ const msg = `Cannot find module '${launcher}'. Did you you install the '${launcher}' package?`;
+ err.message = isAtHome()
+ ? `${msg} The '${launcher}' package is automatically bundled when using ${apifyImageName} Base image.`
+ : msg;
+ }
+
+ throw err;
+ }
+ }
+
+ /**
+ * @param {BrowserLaunchContext} launchContext
+ * All `BrowserLauncher` parameters are passed via an launchContext object.
+ */
+ constructor(launchContext) {
+ ow(launchContext, 'BrowserLauncherOptions', ow.object.exactShape(BrowserLauncher.optionsShape));
+
+ const {
+ launcher,
+ proxyUrl,
+ useChrome,
+ launchOptions = {},
+ } = launchContext;
+
+ this._validateProxyUrl(proxyUrl);
+
+ this.launcher = launcher;
+ this.proxyUrl = proxyUrl;
+ this.useChrome = useChrome;
+ this.launchOptions = launchOptions;
+
+ this.Plugin = null; // to be provided by child classes;
+ }
+
+ /**
+ * @returns {BrowserPlugin}
+ * @ignore
+ */
+ createBrowserPlugin() {
+ return new this.Plugin(
+ this.launcher,
+ {
+ proxyUrl: this.proxyUrl,
+ launchOptions: this.createLaunchOptions(),
+ },
+ );
+ }
+
+ /**
+ * Launches a browser instance based on the plugin.
+ * @returns {object} Browser instance.
+ */
+ async launch() {
+ const plugin = this.createBrowserPlugin();
+ const context = await plugin.createLaunchContext();
+
+ const browser = await plugin.launch(context);
+
+ return browser;
+ }
+
+ /**
+ * @returns {object}
+ */
+ createLaunchOptions() {
+ const launchOptions = {
+ ...this.launchOptions,
+ };
+
+ if (launchOptions.headless == null) {
+ launchOptions.headless = this._getDefaultHeadlessOption();
+ }
+
+ if (this.useChrome && !launchOptions.executablePath) {
+ launchOptions.executablePath = this._getChromeExecutablePath();
+ }
+
+ return launchOptions;
+ }
+
+ /**
+ * @returns {boolean}
+ * @private
+ */
+ _getDefaultHeadlessOption() {
+ return process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';
+ }
+
+ /**
+ * @returns {string}
+ * @private
+ */
+ _getChromeExecutablePath() {
+ return process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();
+ }
+
+ /**
+ *
+ * @param {string} proxyUrl
+ * @private
+ */
+ _validateProxyUrl(proxyUrl) {
+ if (!proxyUrl) {
+ return;
+ }
+
+ const parsedProxyUrl = new URL(proxyUrl);
+ if (!parsedProxyUrl.host || !parsedProxyUrl.port) {
+ throw new Error('Invalid "proxyUrl" option: both hostname and port must be provided.');
+ }
+ if (!/^(http|https|socks4|socks5)$/.test(parsedProxyUrl.protocol.replace(':', ''))) {
+ throw new Error(`Invalid "proxyUrl" option: Unsupported scheme (${parsedProxyUrl.protocol.replace(':', '')}).`);
+ }
+ }
+}
diff --git a/src/playwright.js b/src/browser_launchers/playwright_launcher.js
similarity index 53%
rename from src/playwright.js
rename to src/browser_launchers/playwright_launcher.js
index 041277b7c..39f10cf12 100644
--- a/src/playwright.js
+++ b/src/browser_launchers/playwright_launcher.js
@@ -1,62 +1,6 @@
-import { URL } from 'url';
import ow from 'ow';
-
-import { ENV_VARS } from 'apify-shared/consts';
import { PlaywrightPlugin } from 'browser-pool';
-import { Browser } from 'puppeteer'; // eslint-disable-line no-unused-vars
-import { getTypicalChromeExecutablePath, isAtHome } from './utils';
-import applyStealthToBrowser, { StealthOptions } from './stealth/stealth'; // eslint-disable-line no-unused-vars,import/named
-
-/**
- * Requires `playwright` browserType, uses a replacement or throws meaningful error if not installed.
- *
- * @param {(string|Object)} launcher
- * @ignore
- */
-export function getPlaywrightLauncherOrThrow(launcher = require('playwright').chromium) { // eslint-disable-line
- if (typeof launcher === 'object') return launcher;
- try {
- // This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)
- return require(launcher); // eslint-disable-line
- } catch (err) {
- if (err.code === 'MODULE_NOT_FOUND') {
- const msg = `Cannot find module '${launcher}'. Did you you install the '${launcher}' package?`;
- err.message = isAtHome()
- ? `${msg} The 'playwright' package is automatically bundled when using apify/actor-node-chrome-* Base image.`
- : msg;
- }
-
- throw err;
- }
-}
-
-/**
- *@ignore
- */
-export function getDefaultHeadlessOption() {
- return process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';
-}
-
-/**
- *@ignore
- */
-export function getChromeExecutablePath() {
- return process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();
-}
-
-export function apifyOptionsToLaunchOptions(launchContext) {
- const { launchOptions = {}, useChrome } = launchContext;
-
- if (launchOptions.headless == null) {
- launchOptions.headless = getDefaultHeadlessOption();
- }
-
- if (useChrome && !launchOptions.executablePath) {
- launchOptions.executablePath = getChromeExecutablePath();
- }
-
- return launchOptions;
-}
+import BrowserLauncher from './browser_launcher';
/**
* Apify extends the launch options of Playwright.
@@ -74,16 +18,45 @@ export function apifyOptionsToLaunchOptions(launchContext) {
* Example: `http://bob:[email protected]:1234`.
* @property {boolean} [useChrome=false]
* If `true` and `executablePath` is not set,
- * Puppeteer will launch full Google Chrome browser available on the machine
+ * Playwright will launch full Google Chrome browser available on the machine
* rather than the bundled Chromium. The path to Chrome executable
* is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
* or defaults to the typical Google Chrome executable location specific for the operating system.
* By default, this option is `false`.
- * @property {(string|Object)} [launcher]
+ * @property {Object} [launcher]
* By default this function uses require("playwright").chromium`.
* If you want to use a different browser you can pass it by this property as `require("playwright").firefox
*/
+/**
+ * `PlaywrightLauncher` is based on the `BrowserLauncher`. It launches `playwright` browser instance.
+ */
+export class PlaywrightLauncher extends BrowserLauncher {
+ static optionsShape = {
+ ...BrowserLauncher.optionsShape,
+ launcher: ow.optional.object,
+ }
+
+ /**
+ * @param {PlaywrightLaunchContext} launchContext
+ * All `PlaywrightLauncher` parameters are passed via this launchContext object.
+ */
+ constructor(launchContext = {}) {
+ ow(launchContext, 'PlaywrightLauncherOptions', ow.object.exactShape(PlaywrightLauncher.optionsShape));
+
+ const {
+ launcher = BrowserLauncher.requireLauncherOrThrow('playwright', 'apify/actor-node-playwright-*').chromium,
+ } = launchContext;
+
+ super({
+ ...launchContext,
+ launcher,
+ });
+
+ this.Plugin = PlaywrightPlugin;
+ }
+}
+
/**
* Launches headless browsers using Playwright pre-configured to work within the Apify platform.
* The function has the same return value as `browserType.launch()`.
@@ -122,38 +95,8 @@ export function apifyOptionsToLaunchOptions(launchContext) {
* @name launchPlaywright
* @function
*/
-export const launchPlaywright = async (launchContext = {}) => {
- ow(launchContext, ow.object.partialShape({
- proxyUrl: ow.optional.string.url,
- launcher: ow.optional.any(ow.string, ow.object),
- useChrome: ow.optional.boolean,
-
- }));
-
- const {
- proxyUrl,
- launcher,
- } = launchContext;
-
- if (proxyUrl) {
- const parsedProxyUrl = new URL(proxyUrl);
- if (!parsedProxyUrl.host || !parsedProxyUrl.port) {
- throw new Error('Invalid "proxyUrl" option: both hostname and port must be provided.');
- }
- if (!/^(http|https|socks4|socks5)$/.test(parsedProxyUrl.protocol.replace(':', ''))) {
- throw new Error(`Invalid "proxyUrl" option: Unsupported scheme (${parsedProxyUrl.protocol.replace(':', '')}).`);
- }
- }
- const plugin = new PlaywrightPlugin(
- getPlaywrightLauncherOrThrow(launcher),
- {
- proxyUrl,
- launchOptions: apifyOptionsToLaunchOptions(launchContext),
- },
- );
- const context = await plugin.createLaunchContext();
-
- const browser = await plugin.launch(context);
+export const launchPlaywright = async (launchContext) => {
+ const playwrightLauncher = new PlaywrightLauncher(launchContext);
- return browser;
+ return playwrightLauncher.launch();
};
diff --git a/src/puppeteer.js b/src/browser_launchers/puppeteer_launcher.js
similarity index 52%
rename from src/puppeteer.js
rename to src/browser_launchers/puppeteer_launcher.js
index 6e94810ab..2138fbb9f 100644
--- a/src/puppeteer.js
+++ b/src/browser_launchers/puppeteer_launcher.js
@@ -1,90 +1,15 @@
import ow from 'ow';
-
-import { ENV_VARS } from 'apify-shared/consts';
import { PuppeteerPlugin } from 'browser-pool';
-import { Browser } from 'puppeteer'; // eslint-disable-line no-unused-vars
-import { DEFAULT_USER_AGENT } from './constants';
-import { getTypicalChromeExecutablePath, isAtHome } from './utils';
-import applyStealthToBrowser, { StealthOptions } from './stealth/stealth'; // eslint-disable-line no-unused-vars,import/named
+import BrowserLauncher from './browser_launcher';
+import { isAtHome } from '../utils';
+import { DEFAULT_USER_AGENT } from '../constants';
+
+import applyStealthToBrowser, { StealthOptions } from '../stealth/stealth'; // eslint-disable-line no-unused-vars,import/named
const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
width: 1366,
height: 768,
};
-
-/**
- * Requires `puppeteer` package, uses a replacement or throws meaningful error if not installed.
- *
- * @param {(string|Object)} launcher
- * @ignore
- */
-export function getPuppeteerOrThrow(launcher = 'puppeteer') {
- if (typeof puppeteerModule === 'object') return launcher;
- try {
- // This is an optional dependency because it is quite large, only require it when used (ie. image with Chrome)
- return require(launcher); // eslint-disable-line
- } catch (err) {
- if (err.code === 'MODULE_NOT_FOUND') {
- const msg = `Cannot find module '${launcher}'. Did you you install the '${launcher}' package?`;
- err.message = isAtHome()
- ? `${msg} The 'puppeteer' package is automatically bundled when using apify/actor-node-chrome-* Base image.`
- : msg;
- }
-
- throw err;
- }
-}
-
-/**
- *@ignore
- */
-export function getDefaultHeadlessOption() {
- return process.env[ENV_VARS.HEADLESS] === '1' && process.env[ENV_VARS.XVFB] !== '1';
-}
-
-/**
- *@ignore
- */
-export function getChromeExecutablePath() {
- return process.env[ENV_VARS.CHROME_EXECUTABLE_PATH] || getTypicalChromeExecutablePath();
-}
-
-export function apifyOptionsToLaunchOptions(launchContext) {
- const { launchOptions = {}, useChrome } = launchContext;
-
- launchOptions.args = launchOptions.args || [];
- // Add --no-sandbox for Platform, because running Chrome in Docker
- // is a very complex problem and most likely requires sys admin privileges,
- // which is a larger security concern than --no-sandbox itself.
- // TODO Find if the arg has any impact on browser detection.
- if (isAtHome()) launchOptions.args.push('--no-sandbox');
-
- if (launchOptions.headless == null) {
- launchOptions.headless = getDefaultHeadlessOption();
- }
-
- if (useChrome && !launchOptions.executablePath) {
- launchOptions.executablePath = getChromeExecutablePath();
- }
-
- if (launchOptions.defaultViewport === undefined) {
- launchOptions.defaultViewport = LAUNCH_PUPPETEER_DEFAULT_VIEWPORT;
- }
-
- // When User-Agent is not set and we're using Chromium or headless mode,
- // it is better to use DEFAULT_USER_AGENT to reduce chance of detection
- let { userAgent } = launchContext;
- if (!userAgent && (!launchOptions.executablePath || launchOptions.headless)) {
- userAgent = DEFAULT_USER_AGENT;
- }
-
- if (userAgent) {
- launchOptions.args.push(`--user-agent=${userAgent}`);
- }
-
- return launchOptions;
-}
-
/**
* Apify extends the launch options of Puppeteer.
* You can use any of the Puppeteer compatible
@@ -110,9 +35,8 @@ export function apifyOptionsToLaunchOptions(launchContext) {
* is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
* or defaults to the typical Google Chrome executable location specific for the operating system.
* By default, this option is `false`.
- * @property {(string|Object)} [launcher]
- * Either a require path (`string`) to a package to be used instead of default `puppeteer`,
- * or an already required module (`Object`). This enables usage of various Puppeteer
+ * @property {Object} [launcher]
+ * Already required module (`Object`). This enables usage of various Puppeteer
* wrappers such as `puppeteer-extra`.
*
* Take caution, because it can cause all kinds of unexpected errors and weird behavior.
@@ -125,6 +49,80 @@ export function apifyOptionsToLaunchOptions(launchContext) {
* For these settings to take effect `stealth` must be set to true
*/
+/**
+ * `PuppeteerLauncher` is based on the `BrowserLauncher`. It launches `puppeteer` browser instance.
+ */
+export class PuppeteerLauncher extends BrowserLauncher {
+ static optionsShape = {
+ ...BrowserLauncher.optionsShape,
+ launcher: ow.optional.object,
+ userAgent: ow.optional.string,
+ stealth: ow.optional.boolean,
+ stealthOptions: ow.optional.object,
+ }
+
+ /**
+ * @param {PuppeteerLaunchContext} launchContext
+ * All `PuppeteerLauncher` parameters are passed via an launchContext object.
+ */
+ constructor(launchContext = {}) {
+ ow(launchContext, 'PuppeteerLauncher', ow.object.exactShape(PuppeteerLauncher.optionsShape));
+
+ const {
+ launcher = BrowserLauncher.requireLauncherOrThrow('puppeteer', 'apify/actor-node-chrome-*'),
+ userAgent,
+ stealth,
+ stealthOptions,
+ ...browserLauncherOptions
+ } = launchContext;
+
+ super({
+ ...browserLauncherOptions,
+ launcher,
+ });
+
+ this.userAgent = userAgent;
+ this.stealth = stealth;
+ this.stealthOptions = stealthOptions;
+
+ this.Plugin = PuppeteerPlugin;
+ }
+
+ async launch() {
+ const browser = await super.launch();
+
+ if (this.stealth) {
+ applyStealthToBrowser(browser, this.stealthOptions);
+ }
+
+ return browser;
+ }
+
+ createLaunchOptions() {
+ const launchOptions = super.createLaunchOptions();
+ launchOptions.args = launchOptions.args || [];
+
+ if (isAtHome()) launchOptions.args.push('--no-sandbox');
+
+ if (launchOptions.defaultViewport === undefined) {
+ launchOptions.defaultViewport = LAUNCH_PUPPETEER_DEFAULT_VIEWPORT;
+ }
+
+ // When User-Agent is not set and we're using Chromium or headless mode,
+ // it is better to use DEFAULT_USER_AGENT to reduce chance of detection
+ let { userAgent } = this;
+ if (!userAgent && (!launchOptions.executablePath || launchOptions.headless)) {
+ userAgent = DEFAULT_USER_AGENT;
+ }
+
+ if (userAgent) {
+ launchOptions.args.push(`--user-agent=${userAgent}`);
+ }
+
+ return launchOptions;
+ }
+}
+
/**
* Launches headless Chrome using Puppeteer pre-configured to work within the Apify platform.
* The function has the same argument and the return value as `puppeteer.launch()`.
@@ -164,57 +162,17 @@ export function apifyOptionsToLaunchOptions(launchContext) {
* For an example of usage, see the [Synchronous run Example](../examples/synchronous-run)
* or the [Puppeteer proxy Example](../examples/puppeteer-with-proxy)
*
- * @param {PuppeteerLaunchContext} [options]
- * Optional settings passed to `puppeteer.launch()`. In addition to
- * [Puppeteer's options](https://pptr.dev/#?product=Puppeteer&show=api-puppeteerlaunchoptions)
- * the object may contain our own {@link PuppeteerLaunchContext} that enable additional features.
+ * @param {PuppeteerLaunchContext} [launchContext]
+ * All `PuppeteerLauncher` parameters are passed via an launchContext object.
+ * If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
* @returns {Promise<Browser>}
* Promise that resolves to Puppeteer's `Browser` instance.
* @memberof module:Apify
* @name launchPuppeteer
* @function
*/
-export const launchPuppeteer = async (launchContext = {}) => {
- ow(launchContext, ow.object.partialShape({
- proxyUrl: ow.optional.string.url,
- launcher: ow.optional.any(ow.string, ow.object),
- stealth: ow.optional.boolean,
- stealthOptions: ow.optional.object,
- useChrome: ow.optional.boolean,
- userAgent: ow.optional.string,
- }));
-
- const {
- stealth,
- stealthOptions,
- proxyUrl,
- launcher,
- } = launchContext;
-
- if (proxyUrl) {
- const parsedProxyUrl = new URL(proxyUrl);
- if (!parsedProxyUrl.host || !parsedProxyUrl.port) {
- throw new Error('Invalid "proxyUrl" option: both hostname and port must be provided.');
- }
- if (!/^(http|https|socks4|socks5)$/.test(parsedProxyUrl.protocol.replace(':', ''))) {
- throw new Error(`Invalid "proxyUrl" option: Unsupported scheme (${parsedProxyUrl.protocol.replace(':', '')}).`);
- }
- }
-
- const plugin = new PuppeteerPlugin(
- getPuppeteerOrThrow(launcher),
- {
- proxyUrl,
- launchOptions: apifyOptionsToLaunchOptions(launchContext),
- },
- );
- const context = await plugin.createLaunchContext();
-
- const browser = await plugin.launch(context);
-
- if (stealth) {
- applyStealthToBrowser(browser, stealthOptions);
- }
+export const launchPuppeteer = async (launchContext) => {
+ const puppeteerLauncher = new PuppeteerLauncher(launchContext);
- return browser;
+ return puppeteerLauncher.launch();
};
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.js
index 65c3540c6..4c743944e 100644
--- a/src/crawlers/browser_crawler.js
+++ b/src/crawlers/browser_crawler.js
@@ -1,5 +1,4 @@
import ow from 'ow';
-import _ from 'underscore';
import { BrowserPool } from 'browser-pool'; // eslint-disable-line import/no-duplicates
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { SessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
@@ -365,7 +364,10 @@ class BrowserCrawler extends BasicCrawler {
_enhanceCrawlingContextWithPageInfo(crawlingContext, page) {
crawlingContext.page = page;
- // This is the wierd spam because of browser to proxy not page to proxy.
+ // This switch is because the crawlingContexts are created on per request basis.
+ // However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
+ // We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
+ // which configures proxy and session for every new request
const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page);
crawlingContext.browserController = browserControllerInstance;
@@ -379,8 +381,7 @@ class BrowserCrawler extends BasicCrawler {
* @private
*/
async _handleNavigation(crawlingContext) {
- // @TODO: consider deep clone
- const gotoOptions = _.clone(this.defaultGotoOptions);
+ const gotoOptions = { ...this.defaultGotoOptions };
await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
try {
crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions);
@@ -475,8 +476,7 @@ class BrowserCrawler extends BasicCrawler {
const listener = (session) => {
const { launchContext } = browserController;
if (session.id === launchContext.session.id) {
- launchContext.extend({ sessionRetired: true }); // @TODO: kind of dirty trick done mainly for testing this important feature.
- this.browserPool.retireBrowserController(browserController); //eslint-disable-line
+ this.browserPool.retireBrowserController(browserController);
}
};
diff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.js
index acebde4d1..14a033cc4 100644
--- a/src/crawlers/playwright_crawler.js
+++ b/src/crawlers/playwright_crawler.js
@@ -1,18 +1,9 @@
-import { PlaywrightPlugin } from 'browser-pool';
import ow from 'ow';
import BrowserCrawler from './browser_crawler';
+import { PlaywrightLauncher } from '../browser_launchers/playwright_launcher';
import { gotoExtended } from '../playwright_utils';
-import {
- apifyOptionsToLaunchOptions,
- getPlaywrightLauncherOrThrow,
- PlaywrightLaunchContext, // eslint-disable-line
-} from '../playwright';
-
/**
* @typedef PlaywrightCrawlerOptions
- * @property {object} launcher
- * Playwright launcher used for launching new browsers. For example: `require("playwright").chromium`
- * [Playwright list of available modules](https://playwright.dev/docs/api/class-playwright)
* @property {function} handlePageFunction
* Function that is called to process each request.
* It is passed an object with the following fields:
@@ -32,8 +23,6 @@ import {
* `request` is an instance of the {@link Request} object with details about the URL to open, HTTP method etc.
* `page` is an instance of the `Playwright`
* [`Page`](https://playwright.dev/docs/api/class-page)
- * `browserPool` is an instance of the
- * [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool),
* `browserController` is an instance of the
* [`BrowserController`](https://github.com/apify/browser-pool#browsercontroller),
* `response` is an instance of the `Playwright`
@@ -117,7 +106,7 @@ import {
* @property {number} [handleRequestTimeoutSecs=60]
* Timeout in which the function passed as `handleRequestFunction` needs to finish, in seconds.
* @property {number} [maxRequestRetries=3]
- * Indicates how many times the request is retried if {@link PuppeteerCrawlerOptions.handlePageFunction} fails.
+ * Indicates how many times the request is retried if {@link PlaywrightCrawlerOptions.handlePageFunction} fails.
* @property {number} [maxRequestsPerCrawl]
* Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.
* Always set this value in order to prevent infinite loops in misconfigured crawlers.
@@ -152,35 +141,35 @@ import {
* which downloads the pages using raw HTTP requests and is about 10x faster.
*
* The source URLs are represented using {@link Request} objects that are fed from
- * {@link RequestList} or {@link RequestQueue} instances provided by the {@link BasicCrawlerOptions.requestList}
- * or {@link BasicCrawlerOptions.requestQueue} constructor options, respectively.
+ * {@link RequestList} or {@link RequestQueue} instances provided by the {@link PlaywrightCrawlerOptions.requestList}
+ * or {@link PlaywrightCrawlerOptions.requestQueue} constructor options, respectively.
*
- * If both {@link BasicCrawlerOptions.requestList} and {@link BasicCrawlerOptions.requestQueue} are used,
+ * If both {@link PlaywrightCrawlerOptions.requestList} and {@link PlaywrightCrawlerOptions.requestQueue} are used,
* the instance first processes URLs from the {@link RequestList} and automatically enqueues all of them
* to {@link RequestQueue} before it starts their processing. This ensures that a single URL is not crawled multiple times.
*
* The crawler finishes when there are no more {@link Request} objects to crawl.
*
- * `PuppeteerCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
+ * `PlaywrightCrawler` opens a new Chrome page (i.e. tab) for each {@link Request} object to crawl
* and then calls the function provided by user as the {@link PlaywrightCrawlerOptions.handlePageFunction} option.
*
* New pages are only opened when there is enough free CPU and memory available,
* using the functionality provided by the {@link AutoscaledPool} class.
- * All {@link AutoscaledPool} configuration options can be passed to the {@link BasicCrawlerOptions.autoscaledPoolOptions}
- * parameter of the `PuppeteerCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
- * {@link AutoscaledPoolOptions} are available directly in the `PuppeteerCrawler` constructor.
+ * All {@link AutoscaledPool} configuration options can be passed to the {@link PlaywrightCrawlerOptions.autoscaledPoolOptions}
+ * parameter of the `PlaywrightCrawler` constructor. For user convenience, the `minConcurrency` and `maxConcurrency`
+ * {@link AutoscaledPoolOptions} are available directly in the `PlaywrightCrawler` constructor.
*
- * Note that the pool of Puppeteer instances is internally managed by the {@link BrowserPool} class.
- * Many constructor options such as {@link PuppeteerPoolOptions.maxOpenPagesPerInstance} or
+ * Note that the pool of Playwright instances is internally managed by the {@link BrowserPool} class.
+ * Many constructor options such as {@link PlaywrightCrawlerOptions.maxOpenPagesPerInstance} or
*
* **Example usage:**
*
* ```javascript
- * const crawler = new Apify.PuppeteerCrawler({
+ * const crawler = new Apify.PlaywrightCrawler({
* requestList,
* handlePageFunction: async ({ page, request }) => {
* // This function is called to extract data from a single web page
- * // 'page' is an instance of Puppeteer.Page with page.goto(request.url) already called
+ * // 'page' is an instance of Playwright.Page with page.goto(request.url) already called
* // 'request' is an instance of Request class with information about the page to load
* await Apify.pushData({
* title: await page.title(),
@@ -238,7 +227,6 @@ class PlaywrightCrawler extends BrowserCrawler {
ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
const {
- launcher,
launchContext = {},
gotoTimeoutSecs,
navigationTimeoutSecs,
@@ -250,15 +238,10 @@ class PlaywrightCrawler extends BrowserCrawler {
throw new Error('PlaywrightCrawlerOptions.launchContext.proxyUrl is not allowed in PlaywrightCrawler.'
+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');
}
+ const playwrightLauncher = new PlaywrightLauncher(launchContext);
browserPoolOptions.browserPlugins = [
- new PlaywrightPlugin(
- // eslint-disable-next-line
- getPlaywrightLauncherOrThrow(launcher),
- {
- launchOptions: apifyOptionsToLaunchOptions(launchContext),
- },
- ),
+ playwrightLauncher.createBrowserPlugin(),
];
super({
@@ -272,7 +255,6 @@ class PlaywrightCrawler extends BrowserCrawler {
this.navigationTimeoutMillis = (navigationTimeoutSecs || gotoTimeoutSecs) * 1000;
this.launchContext = launchContext;
- this.launcher = launcher;
this.defaultGotoOptions = {
timeout: this.navigationTimeoutMillis,
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index c8560a8c0..b2359b183 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -1,9 +1,8 @@
-import { PuppeteerPlugin } from 'browser-pool';
import ow from 'ow';
import BrowserCrawler from './browser_crawler';
import { gotoExtended } from '../puppeteer_utils';
-import { apifyOptionsToLaunchOptions, getPuppeteerOrThrow } from '../puppeteer';
+import { PuppeteerLauncher } from '../browser_launchers/puppeteer_launcher';
import applyStealthToBrowser from '../stealth/stealth';
/**
@@ -247,21 +246,16 @@ class PuppeteerCrawler extends BrowserCrawler {
const {
stealth,
stealthOptions,
- launcher,
} = launchContext;
if (launchContext.proxyUrl) {
throw new Error('PuppeteerCrawlerOptions.launchContext.proxyUrl is not allowed in PuppeteerCrawler.'
+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');
}
+ const puppeteerLauncher = new PuppeteerLauncher(launchContext);
browserPoolOptions.browserPlugins = [
- new PuppeteerPlugin(
- getPuppeteerOrThrow(launcher),
- {
- launchOptions: apifyOptionsToLaunchOptions(launchContext),
- },
- ),
+ puppeteerLauncher.createBrowserPlugin(),
];
super({
diff --git a/src/main.js b/src/main.js
index 1c47dd6b9..bfdc806d7 100644
--- a/src/main.js
+++ b/src/main.js
@@ -7,8 +7,8 @@ import CheerioCrawler from './crawlers/cheerio_crawler';
import { pushData, openDataset } from './storages/dataset';
import events, { initializeEvents, stopEvents } from './events';
import { getValue, setValue, getInput, openKeyValueStore } from './storages/key_value_store';
-import { launchPuppeteer } from './puppeteer';
-import { launchPlaywright } from './playwright';
+import { launchPuppeteer } from './browser_launchers/puppeteer_launcher';
+import { launchPlaywright } from './browser_launchers/playwright_launcher';
import BrowserCrawler from './crawlers/browser_crawler';
import PuppeteerCrawler from './crawlers/puppeteer_crawler';
import PlaywrightCrawler from './crawlers/playwright_crawler';
diff --git a/test/playwright.test.js b/test/browser_launchers/playwright_launcher.test.js
similarity index 98%
rename from test/playwright.test.js
rename to test/browser_launchers/playwright_launcher.test.js
index 0c2682fd3..f1436318c 100644
--- a/test/playwright.test.js
+++ b/test/browser_launchers/playwright_launcher.test.js
@@ -6,8 +6,8 @@ import basicAuthParser from 'basic-auth-parser';
import _ from 'underscore';
import sinon from 'sinon';
import { ENV_VARS } from 'apify-shared/consts';
-import Apify from '../build/index';
-import * as utils from '../build/utils';
+import Apify from '../../build/index';
+import * as utils from '../../build/utils';
let prevEnvHeadless;
let proxyServer;
diff --git a/test/puppeteer.test.js b/test/browser_launchers/puppeteer_launcher.test.js
similarity index 93%
rename from test/puppeteer.test.js
rename to test/browser_launchers/puppeteer_launcher.test.js
index ca01ad40d..5037ffeeb 100644
--- a/test/puppeteer.test.js
+++ b/test/browser_launchers/puppeteer_launcher.test.js
@@ -6,8 +6,8 @@ import basicAuthParser from 'basic-auth-parser';
import _ from 'underscore';
import sinon from 'sinon';
import { ENV_VARS } from 'apify-shared/consts';
-import Apify from '../build/index';
-import * as utils from '../build/utils';
+import Apify from '../../build/index';
+import * as utils from '../../build/utils';
let prevEnvHeadless;
let proxyServer;
@@ -200,26 +200,14 @@ describe('Apify.launchPuppeteer()', () => {
}
});
- test('puppeteerModule option works with type string', async () => {
- let browser;
- try {
- browser = await Apify.launchPuppeteer({
- puppeteerModule: 'puppeteer',
- launchOptions: { headless: true },
- });
- } finally {
- if (browser) await browser.close();
- }
- });
-
- test('puppeteerModule option works with type object', async () => {
+ test('launcher option works with type object', async () => {
const someProps = { foo: 'bar' };
// Make it circular for extra security.
someProps.props = someProps;
let browser;
try {
browser = await Apify.launchPuppeteer({
- puppeteerModule: {
+ launcher: {
launch: async () => {},
someProps,
},
diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.js
index 80257b77f..747181f76 100644
--- a/test/crawlers/browser_crawler.test.js
+++ b/test/crawlers/browser_crawler.test.js
@@ -442,7 +442,12 @@ describe('BrowserCrawler', () => {
{ url: 'http://example.com/?q=1' },
],
});
- let controller;
+ let resolve;
+
+ const retirementPromise = new Promise((r) => {
+ resolve = r;
+ });
+ let called = false;
const browserCrawler = new Apify.BrowserCrawler({
browserPoolOptions: {
browserPlugins: [puppeteerPlugin],
@@ -450,18 +455,24 @@ describe('BrowserCrawler', () => {
requestList,
useSessionPool: true,
handlePageFunction: async () => {
- return Promise.resolve();
+ await retirementPromise;
},
maxRequestRetries: 1,
- gotoFunction: async ({ session, browserController }) => {
- controller = browserController;
+ gotoFunction: async ({ session, crawler, page, request }) => {
+ crawler.browserPool.on('browserRetired', () => {
+ resolve();
+ called = true;
+ });
+
session.retire();
+ return page.goto(request.url);
},
});
await requestList.initialize();
await browserCrawler.run();
- expect(controller.launchContext.sessionRetired).toBeTruthy();
+
+ expect(called).toBeTruthy();
});
test('should remove browser listener on session pool', async () => {
diff --git a/test/crawlers/playwright_crawler.test.js b/test/crawlers/playwright_crawler.test.js
index e50affc98..eb7adb560 100644
--- a/test/crawlers/playwright_crawler.test.js
+++ b/test/crawlers/playwright_crawler.test.js
@@ -52,7 +52,9 @@ describe('PlaywrightCrawler', () => {
};
const playwrightCrawler = new Apify.PlaywrightCrawler({
- launcher: playwright[browser],
+ launchContext: {
+ launcher: playwright[browser],
+ },
requestList: requestListLarge,
minConcurrency: 1,
maxConcurrency: 1,