Skip to content

A platform for community discussion. Free, open, simple.

DEV: Stop reading `node_modules` at runtime (#44281)

David Taylorcbb6c885d504
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 41da2249431..3fef8e2e4fe 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -271,6 +271,28 @@ jobs:
exit 1
fi
+ - name: Install playwright
+ if: matrix.build_type == 'system'
+ run: pnpm playwright-install
+
+ - name: Precompile the JS bundles
+ if: matrix.build_type == 'backend' || matrix.build_type == 'system'
+ run: bin/rake assets:precompile:asset_processor assets:precompile:pretty_text
+
+ # TODO: drop hard-coded pnpm 12 once package.json is updated
+ - name: Deploy Playwright outside node_modules
+ if: matrix.build_type == 'system'
+ env:
+ npm_config_manage_package_manager_versions: "false"
+ run: |
+ npx -y [email protected] deploy --filter @discourse/playwright-runner --prod tmp/playwright
+ echo "PLAYWRIGHT_CLI_EXECUTABLE_PATH=$PWD/tmp/playwright/node_modules/.bin/playwright" >> $GITHUB_ENV
+
+ # Drop node_modules to match production
+ - name: Remove node_modules
+ if: matrix.build_type == 'backend' || matrix.build_type == 'system'
+ run: find . -name node_modules -type d -prune -not -path './tmp/*' -print0 | xargs -0 rm -rf
+
- name: Core RSpec
if: matrix.build_type == 'backend' && matrix.target == 'core'
run: LD_PRELOAD="$RUBY_ALLOCATOR" bin/turbo_rspec --use-runtime-info --format=progress
@@ -311,10 +333,6 @@ jobs:
name: ember-exam-execution-${{ matrix.target }}-${{ matrix.browser }}-frontend-${{ hashFiles('./frontend/discourse/test-execution-*.json') }}
path: ./frontend/discourse/test-execution-*.json
- - name: Install playwright
- if: matrix.build_type == 'system'
- run: pnpm playwright-install
-
- name: Core System Tests
if: matrix.build_type == 'system' && matrix.target == 'core'
env:
diff --git a/.gitignore b/.gitignore
index f5d2318f66e..8f57bb53f57 100644
--- a/.gitignore
+++ b/.gitignore
@@ -85,6 +85,7 @@
/spec/fixtures/plugins/my_plugin/auto_generated
/vendor/bundle
+/vendor/runtime_node_modules
/vendor/data/GeoLite2-City.mmdb
/vendor/data/GeoLite2-ASN.mmdb
diff --git a/.npmrc b/.npmrc
index 5ff9ef31a66..53271dd65f8 100644
--- a/.npmrc
+++ b/.npmrc
@@ -1,4 +1,2 @@
+# blocks non-pnpm package-managers
engine-strict = true
-auto-install-peers = false
-ignore-workspace-cycles = true
-update-notifier = false
diff --git a/app/views/qunit/qunit.html.erb b/app/views/qunit/qunit.html.erb
index a43beec4608..e2cdc45bc6e 100644
--- a/app/views/qunit/qunit.html.erb
+++ b/app/views/qunit/qunit.html.erb
@@ -54,7 +54,7 @@
<link rel="canonical" href="/">
<style>
- <%= File.read(Rails.root.join("frontend/discourse/node_modules/qunit/qunit/qunit.css")).html_safe %>
+ <%= File.read(VendoredAssets.path("qunit/qunit.css")).html_safe %>
#ember-testing * {
-webkit-transition: none !important;
diff --git a/config/application.rb b/config/application.rb
index d259b607d5c..a41ca6092d5 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -12,6 +12,7 @@ require_relative "../lib/discourse_event"
require_relative "../lib/discourse_plugin_registry"
require_relative "../lib/plugin_gem"
+require_relative "../lib/vendored_assets"
# Global config
require_relative "../app/models/global_setting"
diff --git a/frontend/playwright-runner/package.json b/frontend/playwright-runner/package.json
new file mode 100644
index 00000000000..8310376ff9a
--- /dev/null
+++ b/frontend/playwright-runner/package.json
@@ -0,0 +1,8 @@
+{
+ "name": "@discourse/playwright-runner",
+ "version": "1.0.0",
+ "private": true,
+ "dependencies": {
+ "playwright": "1.62.1"
+ }
+}
diff --git a/lib/asset_processor.rb b/lib/asset_processor.rb
index 95b1245391e..d114f5f8b96 100644
--- a/lib/asset_processor.rb
+++ b/lib/asset_processor.rb
@@ -8,7 +8,7 @@ class AssetProcessor
dir: "tmp/asset-processor",
filename_prefix: "asset-processor",
dependency_globs: %w[
- node_modules/.pnpm/lock.yaml
+ pnpm-lock.yaml
frontend/asset-processor/**/*.{js,mjs}
app/assets/stylesheets/variable-renames.json
frontend/discourse/lib/babel-transform-module-renames.js
diff --git a/lib/discourse_plugin_registry.rb b/lib/discourse_plugin_registry.rb
index 328390c5b1c..b68a7a52e38 100644
--- a/lib/discourse_plugin_registry.rb
+++ b/lib/discourse_plugin_registry.rb
@@ -267,15 +267,14 @@ class DiscoursePluginRegistry
end
VENDORED_CORE_PRETTY_TEXT_MAP = {
- "moment.js" => "frontend/discourse/node_modules/moment/moment.js",
- "moment-timezone.js" =>
- "frontend/discourse/node_modules/moment-timezone/builds/moment-timezone-with-data.js",
+ "moment.js" => "moment/moment.js",
+ "moment-timezone.js" => "moment-timezone/moment-timezone-with-data.js",
}
def self.core_asset_for_name(name)
asset = VENDORED_CORE_PRETTY_TEXT_MAP[name]
raise KeyError, "Asset #{name} not found in #{VENDORED_CORE_PRETTY_TEXT_MAP}" unless asset
- asset
+ "#{VendoredAssets.path(asset)}"
end
def self.clear_modifiers!
diff --git a/lib/highlight_js.rb b/lib/highlight_js.rb
index b16c0556070..87ba219357c 100644
--- a/lib/highlight_js.rb
+++ b/lib/highlight_js.rb
@@ -1,11 +1,14 @@
# frozen_string_literal: true
module HighlightJs
- HIGHLIGHTJS_DIR = "#{Rails.root.join("frontend/discourse/node_modules/@highlightjs/cdn-assets/")}"
VERSION = 1 # bump to invalidate caches following core changes
+ def self.languages_dir
+ @languages_dir ||= "#{VendoredAssets.path("highlightjs/languages")}/"
+ end
+
def self.languages
- langs = Dir.glob(HIGHLIGHTJS_DIR + "languages/*.js").map { |path| File.basename(path)[0..-8] }
+ langs = Dir.glob(languages_dir + "*.js").map { |path| File.basename(path)[0..-8] }
langs.sort
end
@@ -13,7 +16,7 @@ module HighlightJs
def self.bundle(langs)
lang_js =
langs.filter_map do |lang|
- File.read(HIGHLIGHTJS_DIR + "languages/#{lang}.min.js")
+ File.read(languages_dir + "#{lang}.min.js")
rescue Errno::ENOENT
# no file, don't care
end
diff --git a/lib/js_locale_helper.rb b/lib/js_locale_helper.rb
index 827c63a6e1a..da45362cad6 100644
--- a/lib/js_locale_helper.rb
+++ b/lib/js_locale_helper.rb
@@ -276,11 +276,10 @@ module JsLocaleHelper
def self.find_moment_locale(locale_chain, timezone_names: false)
if timezone_names
- path =
- "#{Rails.root.join("node_modules/@discourse/moment-timezone-names-translations/locales")}"
+ path = "#{VendoredAssets.path("moment-timezone-names/locales")}"
type = :moment_js_timezones
else
- path = "#{Rails.root.join("frontend/discourse/node_modules/moment/locale")}"
+ path = "#{VendoredAssets.path("moment/locale")}"
type = :moment_js
end
diff --git a/lib/precompiled_bundle.rb b/lib/precompiled_bundle.rb
index 418d26b4272..28e649da785 100644
--- a/lib/precompiled_bundle.rb
+++ b/lib/precompiled_bundle.rb
@@ -26,6 +26,7 @@ class PrecompiledBundle
with_lock do
return File.read(cache_path) if File.exist?(cache_path)
+ verify_node_modules!
source = @build.call
FileUtils.mkdir_p(File.dirname(cache_path))
Discourse::Utils.atomic_write_file(cache_path, source)
@@ -36,6 +37,18 @@ class PrecompiledBundle
private
+ def verify_node_modules!
+ installed = Rails.root.join("node_modules/.pnpm/lock.yaml")
+ lockfile = Rails.root.join("pnpm-lock.yaml")
+
+ return if File.exist?(installed) && File.read(installed) == File.read(lockfile)
+
+ raise <<~MSG.squish
+ Cannot build #{@filename_prefix}: node_modules is missing or out of date.
+ Run `pnpm install`.
+ MSG
+ end
+
def digest
digest = Digest::MD5.new
@dependency_globs.each do |pattern|
diff --git a/lib/pretty_text/core_bundle.rb b/lib/pretty_text/core_bundle.rb
index 963dfb1ac6b..014e7dc5a03 100644
--- a/lib/pretty_text/core_bundle.rb
+++ b/lib/pretty_text/core_bundle.rb
@@ -25,7 +25,7 @@ module PrettyText
filename_prefix: "pretty-text",
dependency_globs:
%w[
- node_modules/.pnpm/lock.yaml
+ pnpm-lock.yaml
frontend/pretty-text-processor/**/*.{js,mjs,cjs,json}
frontend/pretty-text/addon/**/*.js
frontend/discourse-markdown-it/src/**/*.js
diff --git a/lib/vendored_assets.rb b/lib/vendored_assets.rb
new file mode 100644
index 00000000000..b01e1d1038e
--- /dev/null
+++ b/lib/vendored_assets.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# `script/copy_vendored_assets.mjs` copies these dependencies out of `node_modules` during `pnpm install`,
+# so that a production install does not need to keep `node_modules` around.
+module VendoredAssets
+ DIRECTORY = "vendor/runtime_node_modules"
+
+ def self.path(relative_path)
+ Rails.root.join(DIRECTORY, relative_path)
+ end
+end
diff --git a/package.json b/package.json
index 7dfd80f96c1..336f3d57020 100644
--- a/package.json
+++ b/package.json
@@ -18,7 +18,6 @@
"licensee": "^12.0.2",
"lint-to-the-future": "^3.2.0",
"lint-to-the-future-eslint": "^4.0.0",
- "playwright": "1.62.1",
"postcss-selector-parser": "^7.1.6",
"postcss-value-parser": "^4.2.0",
"prettier": "3.8.1",
@@ -42,6 +41,7 @@
"lttf:output": "lint-to-the-future output -o ./lint-progress/",
"lint-progress": "pnpm lttf:output && npx html-pages ./lint-progress --no-cache",
"playwright-install": "playwright install --no-shell chromium",
+ "postinstall": "node script/copy_vendored_assets.mjs",
"build": "pnpm --silent --dir=frontend/discourse build",
"start": "pnpm --silent --dir=frontend/discourse start"
},
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 2be727f63b3..900ee48c71f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -76,9 +76,6 @@ importers:
lint-to-the-future-eslint:
specifier: ^4.0.0
- playwright:
- specifier: 1.62.1
- version: 1.62.1
postcss-selector-parser:
specifier: ^7.1.6
version: 7.1.6
@@ -699,6 +696,12 @@ importers:
specifier: ^6.1.1
version: 6.1.1
+ frontend/playwright-runner:
+ dependencies:
+ playwright:
+ specifier: 1.62.1
+ version: 1.62.1
+
frontend/pretty-text:
dependencies:
discourse:
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 2709fcef030..2bf3a0d3094 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -1,3 +1,10 @@
+engineStrict: true
+publicHoistPattern:
+- playwright
+- playwright-core
+autoInstallPeers: false
+ignoreWorkspaceCycles: true
+updateNotifier: false
strictPeerDependencies: true
strictDepBuilds: true
onlyBuiltDependencies:
@@ -20,6 +27,7 @@ packages:
- "frontend/pretty-text-processor"
- "frontend/asset-processor"
- "frontend/discourse-types"
+- "frontend/playwright-runner"
- "plugins/automation"
- "plugins/boards"
- "plugins/chat"
diff --git a/script/copy_vendored_assets.mjs b/script/copy_vendored_assets.mjs
new file mode 100644
index 00000000000..4774d40937a
--- /dev/null
+++ b/script/copy_vendored_assets.mjs
@@ -0,0 +1,55 @@
+// Copies js files out of node_modules for Rails to use
+
+import { copyFile, mkdir, readdir, rm, stat } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");
+const DESTINATION = join(ROOT, "vendor/runtime_node_modules");
+
+const CONCURRENCY = 16;
+
+const FILES = {
+ "highlightjs/languages":
+ "frontend/discourse/node_modules/@highlightjs/cdn-assets/languages",
+ "moment/moment.js": "frontend/discourse/node_modules/moment/moment.js",
+ "moment/locale": "frontend/discourse/node_modules/moment/locale",
+ "moment-timezone/moment-timezone-with-data.js":
+ "frontend/discourse/node_modules/moment-timezone/builds/moment-timezone-with-data.js",
+ "moment-timezone-names/locales":
+ "node_modules/@discourse/moment-timezone-names-translations/locales",
+ "qunit/qunit.css": "frontend/discourse/node_modules/qunit/qunit/qunit.css",
+};
+
+async function collect(source, target, files) {
+ if (!(await stat(source)).isDirectory()) {
+ files.push([source, target]);
+ return;
+ }
+
+ await mkdir(target, { recursive: true });
+ const entries = await readdir(source);
+ await Promise.all(
+ entries.map((entry) =>
+ collect(join(source, entry), join(target, entry), files)
+ )
+ );
+}
+
+const files = [];
+
+await rm(DESTINATION, { force: true, recursive: true });
+
+for (const [destination, source] of Object.entries(FILES)) {
+ const target = join(DESTINATION, destination);
+ await mkdir(dirname(target), { recursive: true });
+ await collect(join(ROOT, source), target, files);
+}
+
+await Promise.all(
+ Array.from({ length: CONCURRENCY }, async () => {
+ for (let file = files.pop(); file; file = files.pop()) {
+ await copyFile(...file);
+ }
+ })
+);
diff --git a/spec/lib/js_locale_helper_spec.rb b/spec/lib/js_locale_helper_spec.rb
index 8d0dd8fae68..19ea8c913df 100644
--- a/spec/lib/js_locale_helper_spec.rb
+++ b/spec/lib/js_locale_helper_spec.rb
@@ -37,7 +37,7 @@ RSpec.describe JsLocaleHelper do
{ "ar" => "ar", "fa_IR" => "fa" }.each do |locale, moment_locale|
it "formats dates with ASCII digits for #{locale}" do
- v8_ctx.eval(File.read(Rails.root.join("frontend/discourse/node_modules/moment/moment.js")))
+ v8_ctx.eval(File.read(VendoredAssets.path("moment/moment.js")))
v8_ctx.eval("moment.tz = {};")
v8_ctx.eval(JsLocaleHelper.output_locale(locale))
v8_ctx.eval("window._discourse_locale_data.configureMoment();")
diff --git a/spec/lib/plugin/instance_spec.rb b/spec/lib/plugin/instance_spec.rb
index e89a33a72d3..a69c2009ca5 100644
--- a/spec/lib/plugin/instance_spec.rb
+++ b/spec/lib/plugin/instance_spec.rb
@@ -394,7 +394,7 @@ TEXT
plugin.send :register_assets!
expect(DiscoursePluginRegistry.vendored_core_pretty_text.first).to eq(
- "frontend/discourse/node_modules/moment/moment.js",
+ "#{Rails.root.join("vendor/runtime_node_modules/moment/moment.js")}",
)
end
end
@@ -626,12 +626,12 @@ TEXT
expect(locale[:fallbackLocale]).to eq("pt_BR")
expect(locale[:moment_js]).to eq(
- ["pt-br", "#{Rails.root.join("frontend/discourse/node_modules/moment/locale/pt-br.js")}"],
+ ["pt-br", "#{Rails.root.join("vendor/runtime_node_modules/moment/locale/pt-br.js")}"],
)
expect(locale[:moment_js_timezones]).to eq(
[
"pt",
- "#{Rails.root.join("node_modules/@discourse/moment-timezone-names-translations/locales/pt.js")}",
+ "#{Rails.root.join("vendor/runtime_node_modules/moment-timezone-names/locales/pt.js")}",
],
)
expect(locale[:plural]).to be_nil
@@ -647,7 +647,7 @@ TEXT
expect(locale[:fallbackLocale]).to be_nil
expect(locale[:moment_js]).to eq(
- ["tlh", "#{Rails.root.join("frontend/discourse/node_modules/moment/locale/tlh.js")}"],
+ ["tlh", "#{Rails.root.join("vendor/runtime_node_modules/moment/locale/tlh.js")}"],
)
expect(locale[:plural]).to eq(plural.with_indifferent_access)
diff --git a/spec/support/system/driver.rb b/spec/support/system/driver.rb
index 9abc69c7192..aa191323d94 100644
--- a/spec/support/system/driver.rb
+++ b/spec/support/system/driver.rb
@@ -50,7 +50,8 @@ module SystemDrivers
acceptDownloads: true,
downloadsPath: Downloads::FOLDER,
slowMo: ENV["PLAYWRIGHT_SLOW_MO_MS"].to_i, # https://playwright.dev/docs/api/class-browsertype#browser-type-launch-option-slow-mo
- playwright_cli_executable_path: "./node_modules/.bin/playwright",
+ playwright_cli_executable_path:
+ ENV["PLAYWRIGHT_CLI_EXECUTABLE_PATH"].presence || "./node_modules/.bin/playwright",
logger: Logger.new(IO::NULL),
# NOTE: timezoneId is NOT set here because the driver is cached and reused,
# so only the first test's timezone would be applied. Instead, we use CDP
diff --git a/spec/tasks/assets_precompile_spec.rb b/spec/tasks/assets_precompile_spec.rb
deleted file mode 100644
index 895017e1642..00000000000
--- a/spec/tasks/assets_precompile_spec.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe "assets:precompile" do
- describe "assets:precompile:asset_processor" do
- it "compiles the js processor" do
- FileUtils.rm_rf(File.dirname(AssetProcessor::BUNDLE.path))
- Rake::Task["assets:precompile:asset_processor"].actions.first.call
-
- expect(File.exist?(AssetProcessor::BUNDLE.path)).to eq(true)
- end
- end
-end