Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(templates): install browsers on postinstall for playwright (#2104)

Vlad Frangu323768bf3350
diff --git a/packages/cli/src/commands/InstallPlaywrightBrowsersCommand.ts b/packages/cli/src/commands/InstallPlaywrightBrowsersCommand.ts
new file mode 100644
index 000000000..5014afef1
--- /dev/null
+++ b/packages/cli/src/commands/InstallPlaywrightBrowsersCommand.ts
@@ -0,0 +1,44 @@
+/* eslint-disable no-console */
+
+import { execSync } from 'node:child_process';
+
+import ansiColors from 'ansi-colors';
+import type { ArgumentsCamelCase, Argv, CommandModule } from 'yargs';
+
+const envVariable = 'CRAWLEE_SKIP_BROWSER_INSTALL';
+
+interface InstallPlaywrightBrowsersArgs {
+ force?: boolean;
+}
+
+export class InstallPlaywrightBrowsersCommand<T> implements CommandModule<T, InstallPlaywrightBrowsersArgs> {
+ command = 'install-playwright-browsers';
+ describe = 'Installs browsers needed by Playwright for local testing';
+
+ builder = async (args: Argv<T>) => {
+ args.options('force', {
+ alias: 'f',
+ default: false,
+ type: 'boolean',
+ describe: 'Use `--force` to force installation of browsers even if the environment is marked as having them.',
+ });
+
+ return args as Argv<InstallPlaywrightBrowsersArgs>;
+ };
+
+ handler = (args: ArgumentsCamelCase<InstallPlaywrightBrowsersArgs>) => {
+ if (process.env[envVariable]) {
+ if (!args.force) {
+ console.log(ansiColors.green('Browsers are already installed!'));
+ return;
+ }
+
+ console.warn(ansiColors.yellow('Installing Playwright browsers in an environment where browsers have already been installed...'));
+ } else {
+ console.log(ansiColors.green('Installing Playwright browsers...'));
+ }
+
+ // TODO: detect package manager
+ execSync(`npx playwright install`, { stdio: 'inherit' });
+ };
+}
diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts
index 00140c13d..012e96b4f 100755
--- a/packages/cli/src/index.ts
+++ b/packages/cli/src/index.ts
@@ -10,6 +10,8 @@ require('yargonaut')
// eslint-disable-next-line
import { CreateProjectCommand } from './commands/CreateProjectCommand';
// eslint-disable-next-line
+import { InstallPlaywrightBrowsersCommand } from './commands/InstallPlaywrightBrowsersCommand';
+// eslint-disable-next-line
import { RunProjectCommand } from './commands/RunProjectCommand';
// eslint-disable-next-line
@@ -35,6 +37,7 @@ const cli = yargs.scriptName('crawlee')
.alias('h', 'help')
.command(new CreateProjectCommand())
.command(new RunProjectCommand())
+ .command(new InstallPlaywrightBrowsersCommand())
.recommendCommands()
.strict();
diff --git a/packages/templates/templates/playwright-js/package.json b/packages/templates/templates/playwright-js/package.json
index d331e1190..d4ba2e9ea 100644
--- a/packages/templates/templates/playwright-js/package.json
+++ b/packages/templates/templates/playwright-js/package.json
@@ -9,7 +9,8 @@
},
"scripts": {
"start": "node src/main.js",
- "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+ "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",
+ "postinstall": "npx crawlee install-playwright-browsers"
},
"author": "It's not you it's me",
"license": "ISC"
diff --git a/packages/templates/templates/playwright-ts/package.json b/packages/templates/templates/playwright-ts/package.json
index 2c4ed013e..93c629a42 100644
--- a/packages/templates/templates/playwright-ts/package.json
+++ b/packages/templates/templates/playwright-ts/package.json
@@ -18,7 +18,8 @@
"start:prod": "node dist/main.js",
"start:dev": "node --no-warnings=ExperimentalWarning --loader ts-node/esm/transpile-only src/main.ts",
"build": "tsc",
- "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
+ "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1",
+ "postinstall": "npx crawlee install-playwright-browsers"
},
"author": "It's not you it's me",
"license": "ISC"