Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: respect `headless` option in browser crawlers (#1455)

Martin Adámek46f6e4cb542f
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 687fb98f4..3473d336f 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -317,6 +317,7 @@ export abstract class BrowserCrawler<
postNavigationHooks: ow.optional.array,
launchContext: ow.optional.object,
+ headless: ow.optional.boolean,
browserPoolOptions: ow.object,
sessionPoolOptions: ow.optional.object,
persistCookiesPerSession: ow.optional.boolean,
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 764d819c1..cc9b36ea3 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -190,6 +190,7 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri
const {
launchContext = {},
+ headless,
browserPoolOptions = {} as BrowserPoolOptions<PlaywrightPlugin>,
...browserCrawlerOptions
} = options;
@@ -199,6 +200,11 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri
+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');
}
+ if (headless != null) {
+ launchContext.launchOptions ??= {} as LaunchOptions;
+ launchContext.launchOptions.headless = headless;
+ }
+
const playwrightLauncher = new PlaywrightLauncher(launchContext, config);
browserPoolOptions.browserPlugins = [
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index e66a3d710..0a3bca586 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -140,6 +140,7 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee
const {
launchContext = {},
+ headless,
browserPoolOptions = {} as BrowserPoolOptions,
proxyConfiguration,
...browserCrawlerOptions
@@ -150,6 +151,11 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee
+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');
}
+ if (headless != null) {
+ launchContext.launchOptions ??= {} as LaunchOptions;
+ launchContext.launchOptions.headless = headless;
+ }
+
const puppeteerLauncher = new PuppeteerLauncher(launchContext, config);
browserPoolOptions.browserPlugins = [