Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Improve pre-built asset bundles (#38502)

David Taylora6098e00dd38
diff --git a/.github/workflows/publish-assets.yml b/.github/workflows/publish-assets.yml
index 82effc5cb52..cefdde98f9d 100644
--- a/.github/workflows/publish-assets.yml
+++ b/.github/workflows/publish-assets.yml
@@ -6,6 +6,7 @@ on:
- main
- stable
- release/*
+ - precompile-asset-improvements
tags:
- beta
- v*
@@ -20,7 +21,7 @@ jobs:
runs-on: 'cdck-linux-8-core'
container: discourse/discourse_test:release
timeout-minutes: 30
-
+
steps:
- name: Install gh cli
run: |
@@ -46,6 +47,18 @@ jobs:
git config --global user.email "[email protected]"
git config --global user.name "Discourse CI"
+ - name: Symlink vendor/bundle from image
+ run: |
+ ln -s /var/www/discourse/vendor/bundle vendor/bundle
+
+ - name: Setup gems
+ run: |
+ bundle config --local path vendor/bundle
+ bundle config --local deployment true
+ bundle config --local without development
+ bundle install --jobs $(($(nproc) - 1))
+ bundle clean
+
- name: pnpm install
run: pnpm install --frozen-lockfile
diff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rake
index e8b65a9162e..30391b21f50 100644
--- a/lib/tasks/assets.rake
+++ b/lib/tasks/assets.rake
@@ -115,7 +115,6 @@ def log_task_duration(task_description, &task)
task_start = current_timestamp
task.call
STDERR.puts "Done '#{task_description}' : #{(current_timestamp - task_start).round(2)} secs"
- STDERR.puts
end
task "assets:precompile:compress_js": "environment" do
@@ -136,10 +135,15 @@ task "assets:precompile:compress_js": "environment" do
next
end
+ file_path = "public/assets/#{digested_path}"
+
+ if File.exist?("#{file_path}.gz") && File.exist?("#{file_path}.br")
+ STDERR.puts "Already compressed: #{digested_path}"
+ next
+ end
+
proc.call do
log_task_duration(digested_path) do
- STDERR.puts "Compressing: #{digested_path}"
- file_path = "public/assets/#{digested_path}"
gzip(file_path)
brotli(file_path)
end
diff --git a/script/assemble_ember_build.rb b/script/assemble_ember_build.rb
index 435e9c24a33..609217a1b53 100755
--- a/script/assemble_ember_build.rb
+++ b/script/assemble_ember_build.rb
@@ -2,16 +2,20 @@
# frozen_string_literal: true
# rubocop:disable Discourse/NoChdir
+require "bundler/setup"
require "fileutils"
require "tempfile"
require "open3"
require "json"
require "time"
+require "parallel"
+require "etc"
require_relative "../lib/version"
DOWNLOAD_PRE_BUILT_ASSETS = ENV["DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS"] != "0"
DOWNLOAD_TEMP_FILE = "#{__dir__}/../tmp/assets.tar.gz"
+DOWNLOAD_EXTRACT_DIR = "#{__dir__}/../tmp/extracted_assets"
PRE_BUILD_ROOT = "https://get.discourse.org/discourse-assets"
@@ -111,27 +115,25 @@ def download_prebuild_assets!
return false
end
- FileUtils.rm_rf("#{EMBER_APP_DIR}/dist")
- FileUtils.mkdir_p("#{EMBER_APP_DIR}/dist")
+ FileUtils.mkdir_p(DOWNLOAD_EXTRACT_DIR)
begin
- system(
- "tar",
- "--strip-components=1",
- "-xzf",
- DOWNLOAD_TEMP_FILE,
- "-C",
- "#{EMBER_APP_DIR}/dist",
- exception: true,
- )
+ system("tar", "-xzf", DOWNLOAD_TEMP_FILE, "-C", DOWNLOAD_EXTRACT_DIR, exception: true)
rescue RuntimeError => e
log "Failed to extract prebuilt assets: #{e.message}"
return false
end
+ FileUtils.rm_rf("#{EMBER_APP_DIR}/dist")
+ FileUtils.mv("#{DOWNLOAD_EXTRACT_DIR}/core", "#{EMBER_APP_DIR}/dist")
+
+ FileUtils.rm_rf("app/assets/generated")
+ FileUtils.mv("#{DOWNLOAD_EXTRACT_DIR}/plugins", "./app/assets/generated")
+
puts "Prebuilt assets downloaded and extracted successfully."
true
ensure
FileUtils.rm_f(DOWNLOAD_TEMP_FILE) if File.exist?(DOWNLOAD_TEMP_FILE)
+ FileUtils.rm_rf(DOWNLOAD_EXTRACT_DIR) if File.exist?(DOWNLOAD_EXTRACT_DIR)
end
build_cmd = %w[pnpm ember build]
@@ -188,3 +190,18 @@ else
File.write(BUILD_INFO_FILE, JSON.pretty_generate(build_info))
end
end
+
+if ARGV.include?("--compress")
+ files = [*Dir.glob("#{EMBER_APP_DIR}/dist/**/*.js"), *Dir.glob("app/assets/generated/**/*.js")]
+ Parallel.map(files, in_threads: 4) do |file|
+ next if File.exist?("#{file}.gz") && File.exist?("#{file}.br")
+
+ start = Time.now
+
+ system("brotli", "-f", "--quality=11", "-o", "#{file}.br", file, exception: true)
+ IO.popen(["gzip", "-f", "-9", "-c", file], "rb") { |io| File.write("#{file}.gz", io.read) }
+ raise "gzip failed for #{file}" unless $?.success?
+
+ puts "Compressed #{file} in #{(Time.now - start).round(2)}s"
+ end
+end
diff --git a/script/publish_built_assets.rb b/script/publish_built_assets.rb
index 29c785356b4..b940a7fb421 100755
--- a/script/publish_built_assets.rb
+++ b/script/publish_built_assets.rb
@@ -6,9 +6,6 @@ require "fileutils"
require_relative "../lib/version"
require "open3"
-tmp_dir = "#{__dir__}/../tmp/prebuilt_asset_bundles"
-FileUtils.mkdir_p(tmp_dir)
-
core_commit_hash = `git rev-parse HEAD`.strip
version_string = "v#{Discourse::VERSION::STRING}-#{core_commit_hash.slice(0, 8)}"
release_repo = "discourse/discourse-assets"
@@ -24,19 +21,63 @@ if status.success?
exit 0
end
-common_env = { "DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS" => "0", "LOAD_PLUGINS" => "0" }
+COMMON_ENV = {
+ "DISCOURSE_DOWNLOAD_PRE_BUILT_ASSETS" => "0",
+ "LOAD_PLUGINS" => "1",
+ "ROLLUP_PLUGIN_COMPILER" => "1",
+ "SKIP_DB_AND_REDIS" => "1",
+}
+
+Dir.chdir("#{__dir__}/..")
+
+TMP_DIR = "./tmp/prebuilt_asset_bundles"
+FileUtils.rm_rf(TMP_DIR)
+FileUtils.mkdir_p(TMP_DIR)
+
+def bundle(name:, env:, compress:)
+ FileUtils.rm_rf("frontend/discourse/dist")
+ FileUtils.rm_rf("app/assets/generated")
-Dir.chdir("#{__dir__}/../frontend/discourse")
-FileUtils.rm_rf("dist")
+ system(
+ { **COMMON_ENV, **env },
+ "#{__dir__}/assemble_ember_build.rb",
+ "--compress",
+ exception: true,
+ )
-system({ **common_env, "EMBER_ENV" => "production" }, "#{__dir__}/assemble_ember_build.rb")
-FileUtils.rm_rf("dist/assets/plugins")
-system("tar", "-czf", "#{tmp_dir}/production.tar.gz", "dist", exception: true)
+ out_dir = "#{TMP_DIR}/#{name}"
+ FileUtils.mkdir_p(out_dir)
+ FileUtils.mv("frontend/discourse/dist", "#{out_dir}/core")
+ FileUtils.mv("app/assets/generated", "#{out_dir}/plugins")
-FileUtils.rm_rf("dist")
-system({ **common_env, "EMBER_ENV" => "development" }, "#{__dir__}/assemble_ember_build.rb")
-FileUtils.rm_rf("dist/assets/plugins")
-system("tar", "-czf", "#{tmp_dir}/development.tar.gz", "dist", exception: true)
+ system(
+ "tar",
+ "-czf",
+ "#{TMP_DIR}/#{name}.tar.gz",
+ "-C",
+ out_dir,
+ "core",
+ "plugins",
+ exception: true,
+ )
+end
+
+bundle(
+ name: "production",
+ env: {
+ "EMBER_ENV" => "production",
+ "RAILS_ENV" => "production",
+ },
+ compress: true,
+)
+bundle(
+ name: "development",
+ env: {
+ "EMBER_ENV" => "development",
+ "RAILS_ENV" => "development",
+ },
+ compress: false,
+)
puts "Creating release #{version_string} in #{release_repo}..."
system(
@@ -46,8 +87,8 @@ system(
"release",
"create",
version_string,
- "#{tmp_dir}/production.tar.gz",
- "#{tmp_dir}/development.tar.gz",
+ "#{TMP_DIR}/production.tar.gz",
+ "#{TMP_DIR}/development.tar.gz",
"--title",
version_string,
"--notes",
@@ -55,4 +96,4 @@ system(
exception: true,
)
-FileUtils.rm_rf(tmp_dir)
+FileUtils.rm_rf(TMP_DIR)