Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: do not import puppeteer directly from inside browser pool (#2346)
Martin Adámek06ba6ac34b3f
diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.tsindex 4522b9317..cf7b62fd4 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts@@ -1,7 +1,5 @@import { tryCancel } from '@apify/timeout';import type { Cookie } from '@crawlee/types';-// @ts-expect-error not exposed on type level-import { CdpBrowser } from 'puppeteer';import type Puppeteer from 'puppeteer';import type * as PuppeteerTypes from 'puppeteer';@@ -63,6 +61,8 @@ export class PuppeteerController extends BrowserController<}try {+ // @ts-expect-error not exposed on type level+ const { CdpBrowser } = await import('puppeteer');const oldPuppeteerVersion = 'createIncognitoBrowserContext' in CdpBrowser.prototype;const method = oldPuppeteerVersion ? 'createIncognitoBrowserContext' : 'createBrowserContext';const context = await (this.browser as any)[method](contextOptions) as PuppeteerTypes.BrowserContext;diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex 8bae5aefa..ffe182530 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -1,8 +1,6 @@import type { Dictionary } from '@crawlee/types';import type Puppeteer from 'puppeteer';import type * as PuppeteerTypes from 'puppeteer';-// @ts-expect-error not exposed on type level-import { CdpBrowser } from 'puppeteer';import type { PuppeteerNewPageOptions } from './puppeteer-controller';import { PuppeteerController } from './puppeteer-controller';@@ -24,6 +22,8 @@ export class PuppeteerPlugin extends BrowserPlugin<protected async _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.PuppeteerLaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>,): Promise<PuppeteerTypes.Browser> {+ // @ts-expect-error not exposed on type level+ const { CdpBrowser } = await import('puppeteer');const oldPuppeteerVersion = 'createIncognitoBrowserContext' in CdpBrowser.prototype;const {launchOptions,