A platform for community discussion. Free, open, simple.
FEATURE: Improve pre-built asset bundles (#38502)
David Taylora6098e00dd38
diff --git a/.github/workflows/publish-assets.yml b/.github/workflows/publish-assets.ymlindex 82effc5cb52..cefdde98f9d 100644--- a/.github/workflows/publish-assets.yml+++ b/.github/workflows/publish-assets.yml@@ -6,6 +6,7 @@ on:- main- stable- release/*+ - precompile-asset-improvementstags:- beta- v*@@ -20,7 +21,7 @@ jobs:runs-on: 'cdck-linux-8-core'container: discourse/discourse_test:releasetimeout-minutes: 30-+steps:- name: Install gh clirun: |@@ -46,6 +47,18 @@ jobs:git config --global user.email "[email protected]"git config --global user.name "Discourse CI"+ - name: Symlink vendor/bundle from image+ run: |+ ln -s /var/www/discourse/vendor/bundle vendor/bundle++ - name: Setup gems+ run: |+ bundle config --local path vendor/bundle+ bundle config --local deployment true+ bundle config --local without development+ bundle install --jobs $(($(nproc) - 1))+ bundle clean+- name: pnpm installrun: pnpm install --frozen-lockfilediff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rakeindex e8b65a9162e..30391b21f50 100644--- a/lib/tasks/assets.rake+++ b/lib/tasks/assets.rake@@ -115,7 +115,6 @@ def log_task_duration(task_description, &task)task_start = current_timestamptask.callSTDERR.puts "Done '#{task_description}' : #{(current_timestamp - task_start).round(2)} secs"- STDERR.putsendtask "assets:precompile:compress_js": "environment" do@@ -136,10 +135,15 @@ task "assets:precompile:compress_js": "environment" donextend+ file_path = "public/assets/#{digested_path}"++ if File.exist?("#{file_path}.gz") && File.exist?("#{file_path}.br")+ STDERR.puts "Already compressed: #{digested_path}"+ next+ end+proc.call dolog_task_duration(digested_path) do- STDERR.puts "Compressing: #{digested_path}"- file_path = "public/assets/#{digested_path}"gzip(file_path)brotli(file_path)enddiff --git a/script/assemble_ember_build.rb b/script/assemble_ember_build.rbindex 435e9c24a33..609217a1b53 100755--- a/script/assemble_ember_build.rb+++ b/script/assemble_ember_build.rb@@ -2,16 +2,20 @@# frozen_string_literal: true# rubocop:disable Discourse/NoChdir+require "bundler/setup"require "fileutils"require "tempfile"require "open3"require "json"require "time"+require "parallel"+require "etc"require_relative "../lib/version"DOWNLOAD_PRE_BUILT_ASSETS = ENV["DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS"] != "0"DOWNLOAD_TEMP_FILE = "#{__dir__}/../tmp/assets.tar.gz"+DOWNLOAD_EXTRACT_DIR = "#{__dir__}/../tmp/extracted_assets"PRE_BUILD_ROOT = "https://get.discourse.org/discourse-assets"@@ -111,27 +115,25 @@ def download_prebuild_assets!return falseend- FileUtils.rm_rf("#{EMBER_APP_DIR}/dist")- FileUtils.mkdir_p("#{EMBER_APP_DIR}/dist")+ FileUtils.mkdir_p(DOWNLOAD_EXTRACT_DIR)begin- system(- "tar",- "--strip-components=1",- "-xzf",- DOWNLOAD_TEMP_FILE,- "-C",- "#{EMBER_APP_DIR}/dist",- exception: true,- )+ system("tar", "-xzf", DOWNLOAD_TEMP_FILE, "-C", DOWNLOAD_EXTRACT_DIR, exception: true)rescue RuntimeError => elog "Failed to extract prebuilt assets: #{e.message}"return falseend+ FileUtils.rm_rf("#{EMBER_APP_DIR}/dist")+ FileUtils.mv("#{DOWNLOAD_EXTRACT_DIR}/core", "#{EMBER_APP_DIR}/dist")++ FileUtils.rm_rf("app/assets/generated")+ FileUtils.mv("#{DOWNLOAD_EXTRACT_DIR}/plugins", "./app/assets/generated")+puts "Prebuilt assets downloaded and extracted successfully."trueensureFileUtils.rm_f(DOWNLOAD_TEMP_FILE) if File.exist?(DOWNLOAD_TEMP_FILE)+ FileUtils.rm_rf(DOWNLOAD_EXTRACT_DIR) if File.exist?(DOWNLOAD_EXTRACT_DIR)endbuild_cmd = %w[pnpm ember build]@@ -188,3 +190,18 @@ elseFile.write(BUILD_INFO_FILE, JSON.pretty_generate(build_info))endend++if ARGV.include?("--compress")+ files = [*Dir.glob("#{EMBER_APP_DIR}/dist/**/*.js"), *Dir.glob("app/assets/generated/**/*.js")]+ Parallel.map(files, in_threads: 4) do |file|+ next if File.exist?("#{file}.gz") && File.exist?("#{file}.br")++ start = Time.now++ system("brotli", "-f", "--quality=11", "-o", "#{file}.br", file, exception: true)+ IO.popen(["gzip", "-f", "-9", "-c", file], "rb") { |io| File.write("#{file}.gz", io.read) }+ raise "gzip failed for #{file}" unless $?.success?++ puts "Compressed #{file} in #{(Time.now - start).round(2)}s"+ end+enddiff --git a/script/publish_built_assets.rb b/script/publish_built_assets.rbindex 29c785356b4..b940a7fb421 100755--- a/script/publish_built_assets.rb+++ b/script/publish_built_assets.rb@@ -6,9 +6,6 @@ require "fileutils"require_relative "../lib/version"require "open3"-tmp_dir = "#{__dir__}/../tmp/prebuilt_asset_bundles"-FileUtils.mkdir_p(tmp_dir)-core_commit_hash = `git rev-parse HEAD`.stripversion_string = "v#{Discourse::VERSION::STRING}-#{core_commit_hash.slice(0, 8)}"release_repo = "discourse/discourse-assets"@@ -24,19 +21,63 @@ if status.success?exit 0end-common_env = { "DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS" => "0", "LOAD_PLUGINS" => "0" }+COMMON_ENV = {+ "DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS" => "0",+ "LOAD_PLUGINS" => "1",+ "ROLLUP_PLUGIN_COMPILER" => "1",+ "SKIP_DB_AND_REDIS" => "1",+}++Dir.chdir("#{__dir__}/..")++TMP_DIR = "./tmp/prebuilt_asset_bundles"+FileUtils.rm_rf(TMP_DIR)+FileUtils.mkdir_p(TMP_DIR)++def bundle(name:, env:, compress:)+ FileUtils.rm_rf("frontend/discourse/dist")+ FileUtils.rm_rf("app/assets/generated")-Dir.chdir("#{__dir__}/../frontend/discourse")-FileUtils.rm_rf("dist")+ system(+ { **COMMON_ENV, **env },+ "#{__dir__}/assemble_ember_build.rb",+ "--compress",+ exception: true,+ )-system({ **common_env, "EMBER_ENV" => "production" }, "#{__dir__}/assemble_ember_build.rb")-FileUtils.rm_rf("dist/assets/plugins")-system("tar", "-czf", "#{tmp_dir}/production.tar.gz", "dist", exception: true)+ out_dir = "#{TMP_DIR}/#{name}"+ FileUtils.mkdir_p(out_dir)+ FileUtils.mv("frontend/discourse/dist", "#{out_dir}/core")+ FileUtils.mv("app/assets/generated", "#{out_dir}/plugins")-FileUtils.rm_rf("dist")-system({ **common_env, "EMBER_ENV" => "development" }, "#{__dir__}/assemble_ember_build.rb")-FileUtils.rm_rf("dist/assets/plugins")-system("tar", "-czf", "#{tmp_dir}/development.tar.gz", "dist", exception: true)+ system(+ "tar",+ "-czf",+ "#{TMP_DIR}/#{name}.tar.gz",+ "-C",+ out_dir,+ "core",+ "plugins",+ exception: true,+ )+end++bundle(+ name: "production",+ env: {+ "EMBER_ENV" => "production",+ "RAILS_ENV" => "production",+ },+ compress: true,+)+bundle(+ name: "development",+ env: {+ "EMBER_ENV" => "development",+ "RAILS_ENV" => "development",+ },+ compress: false,+)puts "Creating release #{version_string} in #{release_repo}..."system(@@ -46,8 +87,8 @@ system("release","create",version_string,- "#{tmp_dir}/production.tar.gz",- "#{tmp_dir}/development.tar.gz",+ "#{TMP_DIR}/production.tar.gz",+ "#{TMP_DIR}/development.tar.gz","--title",version_string,"--notes",@@ -55,4 +96,4 @@ system(exception: true,)-FileUtils.rm_rf(tmp_dir)+FileUtils.rm_rf(TMP_DIR)