Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(templates): install browsers on postinstall for playwright (#2104)
Vlad Frangu323768bf3350
diff --git a/packages/cli/src/commands/InstallPlaywrightBrowsersCommand.ts b/packages/cli/src/commands/InstallPlaywrightBrowsersCommand.tsnew file mode 100644index 000000000..5014afef1--- /dev/null+++ b/packages/cli/src/commands/InstallPlaywrightBrowsersCommand.ts@@ -0,0 +1,44 @@+/* eslint-disable no-console */++import { execSync } from 'node:child_process';++import ansiColors from 'ansi-colors';+import type { ArgumentsCamelCase, Argv, CommandModule } from 'yargs';++const envVariable = 'CRAWLEE_SKIP_BROWSER_INSTALL';++interface InstallPlaywrightBrowsersArgs {+ force?: boolean;+}++export class InstallPlaywrightBrowsersCommand<T> implements CommandModule<T, InstallPlaywrightBrowsersArgs> {+ command = 'install-playwright-browsers';+ describe = 'Installs browsers needed by Playwright for local testing';++ builder = async (args: Argv<T>) => {+ args.options('force', {+ alias: 'f',+ default: false,+ type: 'boolean',+ describe: 'Use `--force` to force installation of browsers even if the environment is marked as having them.',+ });++ return args as Argv<InstallPlaywrightBrowsersArgs>;+ };++ handler = (args: ArgumentsCamelCase<InstallPlaywrightBrowsersArgs>) => {+ if (process.env[envVariable]) {+ if (!args.force) {+ console.log(ansiColors.green('Browsers are already installed!'));+ return;+ }++ console.warn(ansiColors.yellow('Installing Playwright browsers in an environment where browsers have already been installed...'));+ } else {+ console.log(ansiColors.green('Installing Playwright browsers...'));+ }++ // TODO: detect package manager+ execSync(`npx playwright install`, { stdio: 'inherit' });+ };+}diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.tsindex 00140c13d..012e96b4f 100755--- a/packages/cli/src/index.ts+++ b/packages/cli/src/index.ts@@ -10,6 +10,8 @@ require('yargonaut')// eslint-disable-next-lineimport { CreateProjectCommand } from './commands/CreateProjectCommand';// eslint-disable-next-line+import { InstallPlaywrightBrowsersCommand } from './commands/InstallPlaywrightBrowsersCommand';+// eslint-disable-next-lineimport { RunProjectCommand } from './commands/RunProjectCommand';// eslint-disable-next-line@@ -35,6 +37,7 @@ const cli = yargs.scriptName('crawlee').alias('h', 'help').command(new CreateProjectCommand()).command(new RunProjectCommand())+ .command(new InstallPlaywrightBrowsersCommand()).recommendCommands().strict();diff --git a/packages/templates/templates/playwright-js/package.json b/packages/templates/templates/playwright-js/package.jsonindex d331e1190..d4ba2e9ea 100644--- a/packages/templates/templates/playwright-js/package.json+++ b/packages/templates/templates/playwright-js/package.json@@ -9,7 +9,8 @@},"scripts": {"start": "node src/main.js",- "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"+ "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",+ "postinstall": "npx crawlee install-playwright-browsers"},"author": "It's not you it's me","license": "ISC"diff --git a/packages/templates/templates/playwright-ts/package.json b/packages/templates/templates/playwright-ts/package.jsonindex 2c4ed013e..93c629a42 100644--- a/packages/templates/templates/playwright-ts/package.json+++ b/packages/templates/templates/playwright-ts/package.json@@ -18,7 +18,8 @@"start:prod": "node dist/main.js","start:dev": "node --no-warnings=ExperimentalWarning --loader ts-node/esm/transpile-only src/main.ts","build": "tsc",- "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"+ "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",+ "postinstall": "npx crawlee install-playwright-browsers"},"author": "It's not you it's me","license": "ISC"