A platform for community discussion. Free, open, simple.
DEV: Stop reading `node_modules` at runtime (#44281)
David Taylorcbb6c885d504
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 41da2249431..3fef8e2e4fe 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -271,6 +271,28 @@ jobs:exit 1fi+ - name: Install playwright+ if: matrix.build_type == 'system'+ run: pnpm playwright-install++ - name: Precompile the JS bundles+ if: matrix.build_type == 'backend' || matrix.build_type == 'system'+ run: bin/rake assets:precompile:asset_processor assets:precompile:pretty_text++ # TODO: drop hard-coded pnpm 12 once package.json is updated+ - name: Deploy Playwright outside node_modules+ if: matrix.build_type == 'system'+ env:+ npm_config_manage_package_manager_versions: "false"+ run: |+ npx -y [email protected] deploy --filter @discourse/playwright-runner --prod tmp/playwright+ echo "PLAYWRIGHT_CLI_EXECUTABLE_PATH=$PWD/tmp/playwright/node_modules/.bin/playwright" >> $GITHUB_ENV++ # Drop node_modules to match production+ - name: Remove node_modules+ if: matrix.build_type == 'backend' || matrix.build_type == 'system'+ run: find . -name node_modules -type d -prune -not -path './tmp/*' -print0 | xargs -0 rm -rf+- name: Core RSpecif: matrix.build_type == 'backend' && matrix.target == 'core'run: LD_PRELOAD="$RUBY_ALLOCATOR" bin/turbo_rspec --use-runtime-info --format=progress@@ -311,10 +333,6 @@ jobs:name: ember-exam-execution-${{ matrix.target }}-${{ matrix.browser }}-frontend-${{ hashFiles('./frontend/discourse/test-execution-*.json') }}path: ./frontend/discourse/test-execution-*.json- - name: Install playwright- if: matrix.build_type == 'system'- run: pnpm playwright-install-- name: Core System Testsif: matrix.build_type == 'system' && matrix.target == 'core'env:diff --git a/.gitignore b/.gitignoreindex f5d2318f66e..8f57bb53f57 100644--- a/.gitignore+++ b/.gitignore@@ -85,6 +85,7 @@/spec/fixtures/plugins/my_plugin/auto_generated/vendor/bundle+/vendor/runtime_node_modules/vendor/data/GeoLite2-City.mmdb/vendor/data/GeoLite2-ASN.mmdbdiff --git a/.npmrc b/.npmrcindex 5ff9ef31a66..53271dd65f8 100644--- a/.npmrc+++ b/.npmrc@@ -1,4 +1,2 @@+# blocks non-pnpm package-managersengine-strict = true-auto-install-peers = false-ignore-workspace-cycles = true-update-notifier = falsediff --git a/app/views/qunit/qunit.html.erb b/app/views/qunit/qunit.html.erbindex a43beec4608..e2cdc45bc6e 100644--- a/app/views/qunit/qunit.html.erb+++ b/app/views/qunit/qunit.html.erb@@ -54,7 +54,7 @@<link rel="canonical" href="/"><style>- <%= File.read(Rails.root.join("frontend/discourse/node_modules/qunit/qunit/qunit.css")).html_safe %>+ <%= File.read(VendoredAssets.path("qunit/qunit.css")).html_safe %>#ember-testing * {-webkit-transition: none !important;diff --git a/config/application.rb b/config/application.rbindex d259b607d5c..a41ca6092d5 100644--- a/config/application.rb+++ b/config/application.rb@@ -12,6 +12,7 @@ require_relative "../lib/discourse_event"require_relative "../lib/discourse_plugin_registry"require_relative "../lib/plugin_gem"+require_relative "../lib/vendored_assets"# Global configrequire_relative "../app/models/global_setting"diff --git a/frontend/playwright-runner/package.json b/frontend/playwright-runner/package.jsonnew file mode 100644index 00000000000..8310376ff9a--- /dev/null+++ b/frontend/playwright-runner/package.json@@ -0,0 +1,8 @@+{+ "name": "@discourse/playwright-runner",+ "version": "1.0.0",+ "private": true,+ "dependencies": {+ "playwright": "1.62.1"+ }+}diff --git a/lib/asset_processor.rb b/lib/asset_processor.rbindex 95b1245391e..d114f5f8b96 100644--- a/lib/asset_processor.rb+++ b/lib/asset_processor.rb@@ -8,7 +8,7 @@ class AssetProcessordir: "tmp/asset-processor",filename_prefix: "asset-processor",dependency_globs: %w[- node_modules/.pnpm/lock.yaml+ pnpm-lock.yamlfrontend/asset-processor/**/*.{js,mjs}app/assets/stylesheets/variable-renames.jsonfrontend/discourse/lib/babel-transform-module-renames.jsdiff --git a/lib/discourse_plugin_registry.rb b/lib/discourse_plugin_registry.rbindex 328390c5b1c..b68a7a52e38 100644--- a/lib/discourse_plugin_registry.rb+++ b/lib/discourse_plugin_registry.rb@@ -267,15 +267,14 @@ class DiscoursePluginRegistryendVENDORED_CORE_PRETTY_TEXT_MAP = {- "moment.js" => "frontend/discourse/node_modules/moment/moment.js",- "moment-timezone.js" =>- "frontend/discourse/node_modules/moment-timezone/builds/moment-timezone-with-data.js",+ "moment.js" => "moment/moment.js",+ "moment-timezone.js" => "moment-timezone/moment-timezone-with-data.js",}def self.core_asset_for_name(name)asset = VENDORED_CORE_PRETTY_TEXT_MAP[name]raise KeyError, "Asset #{name} not found in #{VENDORED_CORE_PRETTY_TEXT_MAP}" unless asset- asset+ "#{VendoredAssets.path(asset)}"enddef self.clear_modifiers!diff --git a/lib/highlight_js.rb b/lib/highlight_js.rbindex b16c0556070..87ba219357c 100644--- a/lib/highlight_js.rb+++ b/lib/highlight_js.rb@@ -1,11 +1,14 @@# frozen_string_literal: truemodule HighlightJs- HIGHLIGHTJS_DIR = "#{Rails.root.join("frontend/discourse/node_modules/@highlightjs/cdn-assets/")}"VERSION = 1 # bump to invalidate caches following core changes+ def self.languages_dir+ @languages_dir ||= "#{VendoredAssets.path("highlightjs/languages")}/"+ end+def self.languages- langs = Dir.glob(HIGHLIGHTJS_DIR + "languages/*.js").map { |path| File.basename(path)[0..-8] }+ langs = Dir.glob(languages_dir + "*.js").map { |path| File.basename(path)[0..-8] }langs.sortend@@ -13,7 +16,7 @@ module HighlightJsdef self.bundle(langs)lang_js =langs.filter_map do |lang|- File.read(HIGHLIGHTJS_DIR + "languages/#{lang}.min.js")+ File.read(languages_dir + "#{lang}.min.js")rescue Errno::ENOENT# no file, don't careenddiff --git a/lib/js_locale_helper.rb b/lib/js_locale_helper.rbindex 827c63a6e1a..da45362cad6 100644--- a/lib/js_locale_helper.rb+++ b/lib/js_locale_helper.rb@@ -276,11 +276,10 @@ module JsLocaleHelperdef self.find_moment_locale(locale_chain, timezone_names: false)if timezone_names- path =- "#{Rails.root.join("node_modules/@discourse/moment-timezone-names-translations/locales")}"+ path = "#{VendoredAssets.path("moment-timezone-names/locales")}"type = :moment_js_timezoneselse- path = "#{Rails.root.join("frontend/discourse/node_modules/moment/locale")}"+ path = "#{VendoredAssets.path("moment/locale")}"type = :moment_jsenddiff --git a/lib/precompiled_bundle.rb b/lib/precompiled_bundle.rbindex 418d26b4272..28e649da785 100644--- a/lib/precompiled_bundle.rb+++ b/lib/precompiled_bundle.rb@@ -26,6 +26,7 @@ class PrecompiledBundlewith_lock doreturn File.read(cache_path) if File.exist?(cache_path)+ verify_node_modules!source = @build.callFileUtils.mkdir_p(File.dirname(cache_path))Discourse::Utils.atomic_write_file(cache_path, source)@@ -36,6 +37,18 @@ class PrecompiledBundleprivate+ def verify_node_modules!+ installed = Rails.root.join("node_modules/.pnpm/lock.yaml")+ lockfile = Rails.root.join("pnpm-lock.yaml")++ return if File.exist?(installed) && File.read(installed) == File.read(lockfile)++ raise <<~MSG.squish+ Cannot build #{@filename_prefix}: node_modules is missing or out of date.+ Run `pnpm install`.+ MSG+ end+def digestdigest = Digest::MD5.new@dependency_globs.each do |pattern|diff --git a/lib/pretty_text/core_bundle.rb b/lib/pretty_text/core_bundle.rbindex 963dfb1ac6b..014e7dc5a03 100644--- a/lib/pretty_text/core_bundle.rb+++ b/lib/pretty_text/core_bundle.rb@@ -25,7 +25,7 @@ module PrettyTextfilename_prefix: "pretty-text",dependency_globs:%w[- node_modules/.pnpm/lock.yaml+ pnpm-lock.yamlfrontend/pretty-text-processor/**/*.{js,mjs,cjs,json}frontend/pretty-text/addon/**/*.jsfrontend/discourse-markdown-it/src/**/*.jsdiff --git a/lib/vendored_assets.rb b/lib/vendored_assets.rbnew file mode 100644index 00000000000..b01e1d1038e--- /dev/null+++ b/lib/vendored_assets.rb@@ -0,0 +1,11 @@+# frozen_string_literal: true++# `script/copy_vendored_assets.mjs` copies these dependencies out of `node_modules` during `pnpm install`,+# so that a production install does not need to keep `node_modules` around.+module VendoredAssets+ DIRECTORY = "vendor/runtime_node_modules"++ def self.path(relative_path)+ Rails.root.join(DIRECTORY, relative_path)+ end+enddiff --git a/package.json b/package.jsonindex 7dfd80f96c1..336f3d57020 100644--- a/package.json+++ b/package.json@@ -18,7 +18,6 @@"licensee": "^12.0.2","lint-to-the-future": "^3.2.0","lint-to-the-future-eslint": "^4.0.0",- "playwright": "1.62.1","postcss-selector-parser": "^7.1.6","postcss-value-parser": "^4.2.0","prettier": "3.8.1",@@ -42,6 +41,7 @@"lttf:output": "lint-to-the-future output -o ./lint-progress/","lint-progress": "pnpm lttf:output && npx html-pages ./lint-progress --no-cache","playwright-install": "playwright install --no-shell chromium",+ "postinstall": "node script/copy_vendored_assets.mjs","build": "pnpm --silent --dir=frontend/discourse build","start": "pnpm --silent --dir=frontend/discourse start"},diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 2be727f63b3..900ee48c71f 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -76,9 +76,6 @@ importers:lint-to-the-future-eslint:specifier: ^4.0.0version: 4.0.0([email protected]([email protected]))- playwright:- specifier: 1.62.1- version: 1.62.1postcss-selector-parser:specifier: ^7.1.6version: 7.1.6@@ -699,6 +696,12 @@ importers:specifier: ^6.1.1version: 6.1.1+ frontend/playwright-runner:+ dependencies:+ playwright:+ specifier: 1.62.1+ version: 1.62.1+frontend/pretty-text:dependencies:discourse:diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 2709fcef030..2bf3a0d3094 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -1,3 +1,10 @@+engineStrict: true+publicHoistPattern:+- playwright+- playwright-core+autoInstallPeers: false+ignoreWorkspaceCycles: true+updateNotifier: falsestrictPeerDependencies: truestrictDepBuilds: trueonlyBuiltDependencies:@@ -20,6 +27,7 @@ packages:- "frontend/pretty-text-processor"- "frontend/asset-processor"- "frontend/discourse-types"+- "frontend/playwright-runner"- "plugins/automation"- "plugins/boards"- "plugins/chat"diff --git a/script/copy_vendored_assets.mjs b/script/copy_vendored_assets.mjsnew file mode 100644index 00000000000..4774d40937a--- /dev/null+++ b/script/copy_vendored_assets.mjs@@ -0,0 +1,55 @@+// Copies js files out of node_modules for Rails to use++import { copyFile, mkdir, readdir, rm, stat } from "node:fs/promises";+import { dirname, join } from "node:path";+import { fileURLToPath } from "node:url";++const ROOT = join(dirname(fileURLToPath(import.meta.url)), "..");+const DESTINATION = join(ROOT, "vendor/runtime_node_modules");++const CONCURRENCY = 16;++const FILES = {+ "highlightjs/languages":+ "frontend/discourse/node_modules/@highlightjs/cdn-assets/languages",+ "moment/moment.js": "frontend/discourse/node_modules/moment/moment.js",+ "moment/locale": "frontend/discourse/node_modules/moment/locale",+ "moment-timezone/moment-timezone-with-data.js":+ "frontend/discourse/node_modules/moment-timezone/builds/moment-timezone-with-data.js",+ "moment-timezone-names/locales":+ "node_modules/@discourse/moment-timezone-names-translations/locales",+ "qunit/qunit.css": "frontend/discourse/node_modules/qunit/qunit/qunit.css",+};++async function collect(source, target, files) {+ if (!(await stat(source)).isDirectory()) {+ files.push([source, target]);+ return;+ }++ await mkdir(target, { recursive: true });+ const entries = await readdir(source);+ await Promise.all(+ entries.map((entry) =>+ collect(join(source, entry), join(target, entry), files)+ )+ );+}++const files = [];++await rm(DESTINATION, { force: true, recursive: true });++for (const [destination, source] of Object.entries(FILES)) {+ const target = join(DESTINATION, destination);+ await mkdir(dirname(target), { recursive: true });+ await collect(join(ROOT, source), target, files);+}++await Promise.all(+ Array.from({ length: CONCURRENCY }, async () => {+ for (let file = files.pop(); file; file = files.pop()) {+ await copyFile(...file);+ }+ })+);diff --git a/spec/lib/js_locale_helper_spec.rb b/spec/lib/js_locale_helper_spec.rbindex 8d0dd8fae68..19ea8c913df 100644--- a/spec/lib/js_locale_helper_spec.rb+++ b/spec/lib/js_locale_helper_spec.rb@@ -37,7 +37,7 @@ RSpec.describe JsLocaleHelper do{ "ar" => "ar", "fa_IR" => "fa" }.each do |locale, moment_locale|it "formats dates with ASCII digits for #{locale}" do- v8_ctx.eval(File.read(Rails.root.join("frontend/discourse/node_modules/moment/moment.js")))+ v8_ctx.eval(File.read(VendoredAssets.path("moment/moment.js")))v8_ctx.eval("moment.tz = {};")v8_ctx.eval(JsLocaleHelper.output_locale(locale))v8_ctx.eval("window._discourse_locale_data.configureMoment();")diff --git a/spec/lib/plugin/instance_spec.rb b/spec/lib/plugin/instance_spec.rbindex e89a33a72d3..a69c2009ca5 100644--- a/spec/lib/plugin/instance_spec.rb+++ b/spec/lib/plugin/instance_spec.rb@@ -394,7 +394,7 @@ TEXTplugin.send :register_assets!expect(DiscoursePluginRegistry.vendored_core_pretty_text.first).to eq(- "frontend/discourse/node_modules/moment/moment.js",+ "#{Rails.root.join("vendor/runtime_node_modules/moment/moment.js")}",)endend@@ -626,12 +626,12 @@ TEXTexpect(locale[:fallbackLocale]).to eq("pt_BR")expect(locale[:moment_js]).to eq(- ["pt-br", "#{Rails.root.join("frontend/discourse/node_modules/moment/locale/pt-br.js")}"],+ ["pt-br", "#{Rails.root.join("vendor/runtime_node_modules/moment/locale/pt-br.js")}"],)expect(locale[:moment_js_timezones]).to eq(["pt",- "#{Rails.root.join("node_modules/@discourse/moment-timezone-names-translations/locales/pt.js")}",+ "#{Rails.root.join("vendor/runtime_node_modules/moment-timezone-names/locales/pt.js")}",],)expect(locale[:plural]).to be_nil@@ -647,7 +647,7 @@ TEXTexpect(locale[:fallbackLocale]).to be_nilexpect(locale[:moment_js]).to eq(- ["tlh", "#{Rails.root.join("frontend/discourse/node_modules/moment/locale/tlh.js")}"],+ ["tlh", "#{Rails.root.join("vendor/runtime_node_modules/moment/locale/tlh.js")}"],)expect(locale[:plural]).to eq(plural.with_indifferent_access)diff --git a/spec/support/system/driver.rb b/spec/support/system/driver.rbindex 9abc69c7192..aa191323d94 100644--- a/spec/support/system/driver.rb+++ b/spec/support/system/driver.rb@@ -50,7 +50,8 @@ module SystemDriversacceptDownloads: true,downloadsPath: Downloads::FOLDER,slowMo: ENV["PLAYWRIGHT_SLOW_MO_MS"].to_i, # https://playwright.dev/docs/api/class-browsertype#browser-type-launch-option-slow-mo- playwright_cli_executable_path: "./node_modules/.bin/playwright",+ playwright_cli_executable_path:+ ENV["PLAYWRIGHT_CLI_EXECUTABLE_PATH"].presence || "./node_modules/.bin/playwright",logger: Logger.new(IO::NULL),# NOTE: timezoneId is NOT set here because the driver is cached and reused,# so only the first test's timezone would be applied. Instead, we use CDPdiff --git a/spec/tasks/assets_precompile_spec.rb b/spec/tasks/assets_precompile_spec.rbdeleted file mode 100644index 895017e1642..00000000000--- a/spec/tasks/assets_precompile_spec.rb+++ /dev/null@@ -1,12 +0,0 @@-# frozen_string_literal: true--RSpec.describe "assets:precompile" do- describe "assets:precompile:asset_processor" do- it "compiles the js processor" do- FileUtils.rm_rf(File.dirname(AssetProcessor::BUNDLE.path))- Rake::Task["assets:precompile:asset_processor"].actions.first.call-- expect(File.exist?(AssetProcessor::BUNDLE.path)).to eq(true)- end- end-end