Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Refactoring playwright crawler (#894)
petrpatek9b92313ad3f6
diff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.jsindex 331e5cb4e..d42ac649a 100644--- a/src/crawlers/playwright_crawler.js+++ b/src/crawlers/playwright_crawler.js@@ -2,12 +2,16 @@ import { PlaywrightPlugin } from 'browser-pool';import ow from 'ow';import BrowserCrawler from './browser_crawler';import { gotoExtended } from '../playwright_utils';-import { apifyOptionsToLaunchOptions, getPlaywrightLauncherOrThrow } from '../playwright';+import {+ apifyOptionsToLaunchOptions,+ getPlaywrightLauncherOrThrow,+ LaunchPlaywrightOptions, // eslint-disable-line+} from '../playwright';/*** @typedef PlaywrightCrawlerOptions- * @property {object} playwrightModule- * Playwright browser module used for launching new browsers. For example: `require("playwright").chromium`+ * @property {object} launcher+ * Playwright launcher used for launching new browsers. For example: `require("playwright").chromium`* [Playwright list of available modules](https://playwright.dev/docs/api/class-playwright)* @property {function} handlePageFunction* Function that is called to process each request.@@ -91,9 +95,9 @@ import { apifyOptionsToLaunchOptions, getPlaywrightLauncherOrThrow } from '../pl* };* ]* ```-* @property {object} [launchContext]- * Options used by {@link Apify#launchPlaywright} to start new Playwright instances.- * * @property {number} [handlePageTimeoutSecs=60]+ * @property {LaunchPlaywrightOptions} [launchContext]+ * The same options as used by {@link Apify#launchPlaywright}.+ * @property {number} [handlePageTimeoutSecs=60]* Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.* @property {BrowserPoolOptions} [browserPoolOptions]* Custom options passed to the underlying [`BrowserPool`](https://github.com/apify/browser-pool#BrowserPool) constructor.@@ -224,17 +228,17 @@ class PlaywrightCrawler extends BrowserCrawler {static optionsShape = {...BrowserCrawler.optionsShape,browserPoolOptions: ow.optional.object,- playwrightModule: ow.optional.object,+ launcher: ow.optional.object,gotoTimeoutSecs: ow.optional.number,navigationTimeoutSecs: ow.optional.number,- launchOptions: ow.optional.object,+ launchContext: ow.optional.object,}constructor(options = {}) {ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));const {- playwrightModule,+ launcher,launchContext = {},gotoTimeoutSecs,navigationTimeoutSecs,@@ -245,7 +249,7 @@ class PlaywrightCrawler extends BrowserCrawler {browserPoolOptions.browserPlugins = [new PlaywrightPlugin(// eslint-disable-next-line- getPlaywrightLauncherOrThrow(playwrightModule),+ getPlaywrightLauncherOrThrow(launcher),{launchOptions: apifyOptionsToLaunchOptions(launchContext),},@@ -257,15 +261,13 @@ class PlaywrightCrawler extends BrowserCrawler {browserPoolOptions,});- this.gotoTimeoutMillis = gotoTimeoutSecs * 1000;-if (gotoTimeoutSecs) {this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');}this.navigationTimeoutMillis = (navigationTimeoutSecs || gotoTimeoutSecs) * 1000;this.launchContext = launchContext;- this.playwrightModule = playwrightModule;+ this.launcher = launcher;this.defaultGotoOptions = {timeout: this.navigationTimeoutMillis,diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex d37e21fbe..cfa88b00c 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -276,6 +276,10 @@ class PuppeteerCrawler extends BrowserCrawler {this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');}+ if (gotoTimeoutSecs) {+ this.log.deprecated('Option "gotoTimeoutSecs" is deprecated. Use "navigationTimeoutSecs" instead.');+ }+if (proxyUrl) {this.log.deprecated('options.launchContext.proxyUrl is deprecated use the options.proxyConfiguration instead');}diff --git a/test/crawlers/playwright_crawler.test.js b/test/crawlers/playwright_crawler.test.jsindex 3ddc15d1c..e50affc98 100644--- a/test/crawlers/playwright_crawler.test.js+++ b/test/crawlers/playwright_crawler.test.js@@ -52,7 +52,7 @@ describe('PlaywrightCrawler', () => {};const playwrightCrawler = new Apify.PlaywrightCrawler({- playwrightModule: playwright[browser],+ launcher: playwright[browser],requestList: requestListLarge,minConcurrency: 1,maxConcurrency: 1,