Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: respect `headless` option in browser crawlers (#1455)
Martin Adámek46f6e4cb542f
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 687fb98f4..3473d336f 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -317,6 +317,7 @@ export abstract class BrowserCrawler<postNavigationHooks: ow.optional.array,launchContext: ow.optional.object,+ headless: ow.optional.boolean,browserPoolOptions: ow.object,sessionPoolOptions: ow.optional.object,persistCookiesPerSession: ow.optional.boolean,diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 764d819c1..cc9b36ea3 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -190,6 +190,7 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwriconst {launchContext = {},+ headless,browserPoolOptions = {} as BrowserPoolOptions<PlaywrightPlugin>,...browserCrawlerOptions} = options;@@ -199,6 +200,11 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');}+ if (headless != null) {+ launchContext.launchOptions ??= {} as LaunchOptions;+ launchContext.launchOptions.headless = headless;+ }+const playwrightLauncher = new PlaywrightLauncher(launchContext, config);browserPoolOptions.browserPlugins = [diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex e66a3d710..0a3bca586 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -140,6 +140,7 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppeteeconst {launchContext = {},+ headless,browserPoolOptions = {} as BrowserPoolOptions,proxyConfiguration,...browserCrawlerOptions@@ -150,6 +151,11 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');}+ if (headless != null) {+ launchContext.launchOptions ??= {} as LaunchOptions;+ launchContext.launchOptions.headless = headless;+ }+const puppeteerLauncher = new PuppeteerLauncher(launchContext, config);browserPoolOptions.browserPlugins = [