Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(puppeteer): enable `new` headless mode (#1910)
Martin Adámek7fc999cf4658
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 7f32b04b0..968460ee5 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -249,7 +249,7 @@ export interface BrowserCrawlerOptions<* Whether to run browser in headless mode. Defaults to `true`.* Can be also set via {@apilink Configuration}.*/- headless?: boolean;+ headless?: boolean | 'new' | 'old'; // `new`/`old` are for puppeteer only}/**@@ -326,7 +326,7 @@ export abstract class BrowserCrawler<postNavigationHooks: ow.optional.array,launchContext: ow.optional.object,- headless: ow.optional.boolean,+ headless: ow.optional.any(ow.boolean, ow.string),browserPoolOptions: ow.object,sessionPoolOptions: ow.optional.object,persistCookiesPerSession: ow.optional.boolean,diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex fbc40e565..4a8ee3e87 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -45,8 +45,7 @@ export class PuppeteerPlugin extends BrowserPlugin<}if (launchOptions!.headless === true) {- // to disable deprecation warnings, we should switch to `new` after more testing- launchOptions!.headless = 'old' as unknown as boolean;+ launchOptions!.headless = 'new';}let browser: PuppeteerTypes.Browser;diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 22ea94ac5..d87f5f512 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -223,7 +223,7 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwriif (headless != null) {launchContext.launchOptions ??= {} as LaunchOptions;- launchContext.launchOptions.headless = headless;+ launchContext.launchOptions.headless = headless as boolean;}const playwrightLauncher = new PlaywrightLauncher(launchContext, config);diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex c8074e975..2682e654e 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -160,15 +160,15 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');}- // `browserPlugins` is working when it's not overriden by `launchContext`,- // which for crawlers it is always overriden. Hence the error to use the other option.+ // `browserPlugins` is working when it's not overridden by `launchContext`,+ // which for crawlers it is always overridden. Hence the error to use the other option.if (browserPoolOptions.browserPlugins) {throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');}if (headless != null) {launchContext.launchOptions ??= {} as LaunchOptions;- launchContext.launchOptions.headless = headless;+ launchContext.launchOptions.headless = headless as boolean;}const puppeteerLauncher = new PuppeteerLauncher(launchContext, config);diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts b/packages/puppeteer-crawler/src/internals/puppeteer-launcher.tsindex 59da496c5..4f5f369a6 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-launcher.ts@@ -98,6 +98,11 @@ export class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown>this.Plugin = PuppeteerPlugin;}++ protected override _getDefaultHeadlessOption(): boolean {+ const headless = super._getDefaultHeadlessOption();+ return headless ? 'new' as any : headless;+ }}/**diff --git a/test/core/puppeteer_request_interception.test.ts b/test/core/puppeteer_request_interception.test.tsindex 1d318f60a..dde96259b 100644--- a/test/core/puppeteer_request_interception.test.ts+++ b/test/core/puppeteer_request_interception.test.ts@@ -1,18 +1,13 @@import type { Server } from 'http';-import type { AddressInfo } from 'net';import { sleep } from '@crawlee/utils';import { launchPuppeteer, utils } from 'crawlee';-import express from 'express';import type { HTTPRequest } from 'puppeteer';import { runExampleComServer } from '../shared/_helper';const { addInterceptRequestHandler, removeInterceptRequestHandler } = utils.puppeteer;-// Simple page with image, script and stylesheet links.-let HTML_PAGE = '';-let serverAddress = 'http://localhost:';let port: number;let server: Server;@@ -20,11 +15,6 @@ let server: Server;beforeAll(async () => {[server, port] = await runExampleComServer();serverAddress += port;- HTML_PAGE = `<html><body>- <link rel="stylesheet" type="text/css" href="${serverAddress}/style.css">- <img src="${serverAddress}/image.png" />- <script src="${serverAddress}/script.js" defer="defer">></script>-</body></html>`;});afterAll(() => {@@ -61,7 +51,7 @@ describe('utils.puppeteer.addInterceptRequestHandler|removeInterceptRequestHandl// Save all loaded URLs.page.on('response', (response) => loadedUrls.push(response.url()));- await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });} finally {await browser.close();}@@ -105,7 +95,7 @@ describe('utils.puppeteer.addInterceptRequestHandler|removeInterceptRequestHandlpropagatedUrls.push(request.url());return request.continue();});- await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });} finally {await browser.close();}@@ -242,15 +232,13 @@ describe('utils.puppeteer.removeInterceptRequestHandler()', () => {});// Load with scripts and images disabled.- await page.setContent('<html><body></body></html>');- await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/style.css`,]));// Try it once again.- await page.setContent('<html><body></body></html>');- await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/style.css`,`${serverAddress}/style.css`,@@ -260,8 +248,7 @@ describe('utils.puppeteer.removeInterceptRequestHandler()', () => {await removeInterceptRequestHandler(page, abortImagesHandler);// Try to load once again if images appear there.- await page.setContent('<html><body></body></html>');- await page.setContent(HTML_PAGE, { waitUntil: 'networkidle0' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'networkidle0' });expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/style.css`,`${serverAddress}/style.css`,diff --git a/test/core/puppeteer_utils.test.ts b/test/core/puppeteer_utils.test.tsindex 07f6cc104..4103ba99b 100644--- a/test/core/puppeteer_utils.test.ts+++ b/test/core/puppeteer_utils.test.ts@@ -4,7 +4,6 @@ import path from 'path';import log from '@apify/log';import { KeyValueStore, launchPuppeteer, puppeteerUtils, Request } from '@crawlee/puppeteer';import type { Dictionary } from '@crawlee/utils';-import express from 'express';import type { Browser, Page, ResponseForRequest } from 'puppeteer';import { runExampleComServer } from 'test/shared/_helper';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';@@ -174,13 +173,11 @@ describe('puppeteerUtils', () => {const page = await browser.newPage();await puppeteerUtils.blockRequests(page);page.on('response', (response) => loadedUrls.push(response.url()));- await page.setContent(`<html><body>- <link rel="stylesheet" type="text/css" href="${serverAddress}/style.css">- <img src="${serverAddress}/image.png">- <img src="${serverAddress}/image.gif">- <script src="${serverAddress}/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'load' });- expect(loadedUrls).toEqual([`${serverAddress}/script.js`]);+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'load' });+ expect(loadedUrls).toEqual([+ `${serverAddress}/special/resources`,+ `${serverAddress}/script.js`,+ ]);});test('works with overridden values', async () => {@@ -191,12 +188,8 @@ describe('puppeteerUtils', () => {urlPatterns: ['.css'],});page.on('response', (response) => loadedUrls.push(response.url()));- await page.setContent(`<html><body>- <link rel="stylesheet" type="text/css" href="${serverAddress}/style.css">- <img src="${serverAddress}/image.png">- <img src="${serverAddress}/image.gif">- <script src="${serverAddress}/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'load' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'load' });+expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/image.png`,`${serverAddress}/script.js`,@@ -210,11 +203,7 @@ describe('puppeteerUtils', () => {const page = await browser.newPage();await puppeteerUtils.blockResources(page);page.on('response', (response) => loadedUrls.push(response.url()));- await page.setContent(`<html><body>- <link rel="stylesheet" type="text/css" href="${serverAddress}/style.css">- <img src="${serverAddress}/image.png" />- <script src="${serverAddress}/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'load' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'load' });expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/script.js`,@@ -227,11 +216,7 @@ describe('puppeteerUtils', () => {const page = await browser.newPage();await puppeteerUtils.blockResources(page, ['script']);page.on('response', (response) => loadedUrls.push(response.url()));- await page.setContent(`<html><body>- <link rel="stylesheet" type="text/css" href="${serverAddress}/style.css">- <img src="${serverAddress}/image.png" />- <script src="${serverAddress}/script.js" defer="defer">></script>- </body></html>`, { waitUntil: 'load' });+ await page.goto(`${serverAddress}/special/resources`, { waitUntil: 'load' });expect(loadedUrls).toEqual(expect.arrayContaining([`${serverAddress}/style.css`,diff --git a/test/shared/_helper.ts b/test/shared/_helper.tsindex d75c09be8..f141d738a 100644--- a/test/shared/_helper.ts+++ b/test/shared/_helper.ts@@ -70,6 +70,13 @@ export const responseSamples = {</div></body></html>`,+ resources: `+ <html><body>+ <link rel="stylesheet" type="text/css" href="/style.css">+ <img src="/image.png">+ <img src="/image.gif">+ <script src="/script.js" defer="defer"></script>+ </body></html>`,cacheable: {html: `<!doctype html>@@ -177,6 +184,9 @@ export async function runExampleComServer(): Promise<[Server, number]> {await setTimeout(32000);res.type('html').send('<div>TEST</div>');});+ special.get('/resources', async (_req, res) => {+ res.type('html').send(responseSamples.resources);+ });})();// "cacheable" site with one page, scripts and stylesheets