A platform for community discussion. Free, open, simple.
DEV: Build server-side PrettyText bundle with Rolldown (#41997)
David Taylorcfed6b85b6ce
diff --git a/frontend/pretty-text-processor/build.mjs b/frontend/pretty-text-processor/build.mjsnew file mode 100644index 00000000000..a165fd51fa4--- /dev/null+++ b/frontend/pretty-text-processor/build.mjs@@ -0,0 +1,88 @@+import { rolldown } from "rolldown";+import { importGlobPlugin, viteAliasPlugin } from "rolldown/experimental";++const FRONTEND = new URL("../", import.meta.url).pathname;+const SHIM = (f) => new URL(`./shims/${f}`, import.meta.url).pathname;++// Allowlist of bundleable `discourse` core modules, supplied by+// PrettyText::BUNDLED_DISCOURSE_MODULES in lib/pretty_text.rb.+const DISCOURSE_MODULES = process.argv+ .find((arg) => arg.startsWith("--discourse-modules="))+ ?.slice("--discourse-modules=".length)+ .split(",");++if (!DISCOURSE_MODULES?.length) {+ throw new Error(+ "Missing --discourse-modules=<comma-separated list> (see lib/pretty_text.rb)"+ );+}++const bundle = await rolldown({+ input: new URL("./entry.js", import.meta.url).pathname,+ platform: "browser",+ plugins: [+ importGlobPlugin(),+ viteAliasPlugin({+ entries: [+ { find: "discourse-i18n", replacement: SHIM("i18n.js") },+ { find: "discourse/lib/helpers", replacement: SHIM("helpers.js") },+ {+ find: "discourse/lib/deprecated",+ replacement: SHIM("deprecated.js"),+ },+ ...DISCOURSE_MODULES.map((m) => ({+ find: `discourse/${m}`,+ replacement: `${FRONTEND}discourse/app/${m}.js`,+ })),+ {+ find: /^pretty-text\//,+ replacement: `${FRONTEND}pretty-text/addon/`,+ },+ ],+ }),+ {+ // Exposes the allowlisted modules as a name -> namespace map, so+ // register-modules.js can define() them without duplicating the list.+ name: "discourse-modules-map",+ resolveId(id) {+ if (id === "virtual:discourse-modules") {+ return "\0virtual:discourse-modules";+ }+ },+ load(id) {+ if (id === "\0virtual:discourse-modules") {+ const modules = DISCOURSE_MODULES.filter((m) => m !== "loader");+ return [+ ...modules.map(+ (m, i) => `import * as mod${i} from "discourse/${m}";`+ ),+ "export default {",+ ...modules.map((m, i) => ` "../discourse/app/${m}.js": mod${i},`),+ "};",+ ].join("\n");+ }+ },+ },+ {+ // The `discourse` package's exports map would resolve anything under+ // discourse/*; only imports rewritten by the aliases above are allowed.+ name: "discourse-import-allowlist",+ resolveId(id, importer) {+ if (id.startsWith("discourse/")) {+ throw new Error(+ `"${id}" (imported by ${importer}) is not in PrettyText::BUNDLED_DISCOURSE_MODULES`+ );+ }+ },+ },+ ],+ resolve: { extensions: [".js", ".mjs", ".cjs", ".json"] },+});++const { output } = await bundle.generate({+ format: "iife",+ minify: false,+});++await bundle.close();+process.stdout.write(output[0].code);diff --git a/frontend/pretty-text-processor/entry.js b/frontend/pretty-text-processor/entry.jsnew file mode 100644index 00000000000..a29132b686e--- /dev/null+++ b/frontend/pretty-text-processor/entry.js@@ -0,0 +1,6 @@+import "discourse/loader"; // sets up window.define / window.require — must come first+import { PrettyTextRubyInterface } from "./pretty-text-ruby-interface.js";+import { registerCoreModules } from "./register-modules.js";++registerCoreModules();+globalThis.__PrettyText = PrettyTextRubyInterface;diff --git a/frontend/pretty-text-processor/package.json b/frontend/pretty-text-processor/package.jsonnew file mode 100644index 00000000000..7df081d26b7--- /dev/null+++ b/frontend/pretty-text-processor/package.json@@ -0,0 +1,23 @@+{+ "name": "pretty-text-processor",+ "version": "1.0.0",+ "private": true,+ "description": "Uses rolldown to bundle the PrettyText core for loading into mini-racer",+ "author": "Discourse",+ "license": "GPL-2.0-only",+ "keywords": [],+ "dependencies": {+ "discourse": "workspace:0.0.0",+ "discourse-markdown-it": "workspace:*",+ "markdown-it": "^14.1.1",+ "pretty-text": "workspace:*",+ "rolldown": "1.1.2",+ "xss": "^1.0.15"+ },+ "engines": {+ "node": ">= 20",+ "npm": "please-use-pnpm",+ "yarn": "please-use-pnpm",+ "pnpm": "^10"+ }+}diff --git a/frontend/pretty-text-processor/pretty-text-ruby-interface.js b/frontend/pretty-text-processor/pretty-text-ruby-interface.jsnew file mode 100644index 00000000000..ef53b4cea28--- /dev/null+++ b/frontend/pretty-text-processor/pretty-text-ruby-interface.js@@ -0,0 +1,142 @@+import AllowLister from "pretty-text/allow-lister";+import {+ emojiReplacementRegex,+ performEmojiEscape,+ performEmojiUnescape,+} from "pretty-text/emoji";+import { sanitize as sanitizeHtml } from "pretty-text/sanitizer";+import * as avatarUtils from "discourse/lib/avatar-utils";+import loadPluginFeatures from "discourse/static/markdown-it/features";+import DiscourseMarkdownIt from "discourse-markdown-it";+import { resetTranslationTree } from "discourse-markdown-it/features/emoji";+import { runtime } from "./runtime-state.js";++// Passed to the markdown pipeline as plain callbacks, so functions not methods.+const ruby = globalThis.__Ruby;++function getURLNoCDN(url) {+ const paths = runtime.paths;+ if (!url) {+ return url;+ }+ if (url !== "/" && !/^\/[^\/]/.test(url)) {+ return url;+ }+ if (url.startsWith(`${paths.baseUri}/`) || url === paths.baseUri) {+ return url;+ }+ if (url[0] !== "/") {+ url = "/" + url;+ }+ return paths.baseUri + url;+}++function getURL(url) {+ const paths = runtime.paths;+ url = getURLNoCDN(url);+ if (paths.CDN && /^\\\/[^\\\/]/.test(url)) {+ url = paths.CDN + url;+ } else if (paths.S3CDN) {+ url = url.replace(paths.S3BaseUrl, paths.S3CDN);+ }+ return url;+}++const lookupUploadUrls = (urls) => ruby.lookup_upload_urls(urls);+const getTopicInfo = (i) => ruby.get_topic_info(i);+const hashtagLookup = (slug, cookingUserId, typesInPriorityOrder) =>+ ruby.hashtag_lookup(slug, cookingUserId, typesInPriorityOrder);+const lookupAvatar = (p) =>+ avatarUtils.avatarImg(+ { size: "tiny", avatarTemplate: ruby.avatar_template(p) },+ getURL+ );+const formatUsername = (username) => ruby.format_username(username);+const lookupPrimaryUserGroup = (username) =>+ ruby.lookup_primary_user_group(username);+const getCurrentUser = (userId) => ruby.get_current_user(userId);++let emojiUnicodeReplacer = null;++function buildEmojiUnicodeReplacer(replacements) {+ const regexp = new RegExp(emojiReplacementRegex, "g");+ return function (text) {+ regexp.lastIndex = 0;+ let m;+ while ((m = regexp.exec(text)) !== null) {+ let match = m[0];+ let replacement = replacements[match];+ if (!replacement) {+ match = match.replace(/️$/g, "");+ replacement = replacements[match];+ }+ if (!replacement) {+ continue;+ }+ replacement = ":" + replacement + ":";+ const before = text.charAt(m.index - 1);+ if (!/\B/.test(before)) {+ replacement = "" + replacement;+ }+ text = text.replace(match, replacement);+ }+ text = text.replace(/️/g, "");+ return text;+ };+}++export class PrettyTextRubyInterface {+ static cook(text, optInput) {+ runtime.paths = optInput.paths || {};+ runtime.avatarSizes = optInput.avatar_sizes;++ optInput.getURL = getURL;+ optInput.getCurrentUser = getCurrentUser;+ optInput.lookupAvatar = lookupAvatar;+ optInput.lookupPrimaryUserGroup = lookupPrimaryUserGroup;+ optInput.formatUsername = formatUsername;+ optInput.getTopicInfo = getTopicInfo;+ optInput.hashtagLookup = hashtagLookup;+ optInput.lookupUploadUrls = lookupUploadUrls;+ optInput.emojiUnicodeReplacer = emojiUnicodeReplacer;++ const pt =+ DiscourseMarkdownIt.withCustomFeatures(loadPluginFeatures()).withOptions(+ optInput+ );++ if (optInput.disableSanitizer) {+ pt.disableSanitizer();+ }++ return pt.cook(text);+ }++ static sanitize(html, allowListOptions) {+ return sanitizeHtml(html, new AllowLister(allowListOptions));+ }++ static avatarImg(avatarTemplate, size, paths, avatarSizes) {+ runtime.paths = paths || {};+ runtime.avatarSizes = avatarSizes;+ return avatarUtils.avatarImg({ size, avatarTemplate }, getURL);+ }++ static setUnicode(replacements) {+ emojiUnicodeReplacer = buildEmojiUnicodeReplacer(replacements);+ }++ static resetTranslations() {+ resetTranslationTree();+ }++ static performEmojiUnescape(text, options) {+ runtime.paths = options.paths || {};+ options.getURL = getURL;+ return performEmojiUnescape(text, options);+ }++ static performEmojiEscape(text, options) {+ return performEmojiEscape(text, options);+ }+}diff --git a/frontend/pretty-text-processor/register-modules.js b/frontend/pretty-text-processor/register-modules.jsnew file mode 100644index 00000000000..80b5b2c70cc--- /dev/null+++ b/frontend/pretty-text-processor/register-modules.js@@ -0,0 +1,50 @@+import markdownit from "markdown-it";+import discourseModules from "virtual:discourse-modules";+import xss from "xss";+import * as deprecated from "discourse/lib/deprecated";+import * as helpersShim from "./shims/helpers.js";+import * as i18nShim from "./shims/i18n.js";++// oneboxer and upload-short-url are browser-only; they were never usable in+// the server-side context.+const ptModules = import.meta.glob(+ [+ "../pretty-text/addon/**/*.js",+ "!../pretty-text/addon/oneboxer.js",+ "!../pretty-text/addon/upload-short-url.js",+ ],+ { eager: true }+);+const mdModules = import.meta.glob("../discourse-markdown-it/src/**/*.js", {+ eager: true,+});++function defineMultiple(prefix, base, mods) {+ for (const [key, mod] of Object.entries(mods)) {+ define(+ `${prefix}/${key.slice(base.length).replace(/\.js$/, "")}`,+ () => mod+ );+ }+}++// Registers the core module surface into loader.js so plugins can require() it.+export function registerCoreModules() {+ defineMultiple("pretty-text", "../pretty-text/addon/", ptModules);+ defineMultiple(+ "discourse-markdown-it",+ "../discourse-markdown-it/src/",+ mdModules+ );++ defineMultiple("discourse", "../discourse/app/", discourseModules);+ define("discourse/lib/deprecated", () => deprecated);++ globalThis.I18n = i18nShim.default;+ define("I18n", () => ({ default: i18nShim.default }));+ define("discourse-i18n", () => i18nShim);+ define("discourse/lib/helpers", () => helpersShim);++ define("markdown-it", () => ({ default: markdownit }));+ define("xss", () => ({ default: xss }));+}diff --git a/frontend/pretty-text-processor/runtime-state.js b/frontend/pretty-text-processor/runtime-state.jsnew file mode 100644index 00000000000..c2ea4d89038--- /dev/null+++ b/frontend/pretty-text-processor/runtime-state.js@@ -0,0 +1,2 @@+// Shared with the discourse/lib/helpers shim (reads avatarSizes); set per call.+export const runtime = { paths: {}, avatarSizes: undefined };diff --git a/frontend/pretty-text-processor/shims/deprecated.js b/frontend/pretty-text-processor/shims/deprecated.jsnew file mode 100644index 00000000000..29c717ad211--- /dev/null+++ b/frontend/pretty-text-processor/shims/deprecated.js@@ -0,0 +1,30 @@+// Server-side stand-in for discourse/lib/deprecated; the real one pulls a+// browser-only dependency chain. Reproduces its console.warn message format.+export default function deprecated(msg, options = {}) {+ const { id, since, url } = options;+ const parts = ["DEPRECATION NOTICE:", msg];+ if (since) {+ parts.push(`[deprecated since Discourse ${since}]`);+ }+ if (id) {+ parts.push(`[deprecation id: ${id}]`);+ }+ if (url) {+ parts.push(`[info: ${url}]`);+ }+ // eslint-disable-next-line no-console+ console.warn(parts.join(" "));+}+export function registerDeprecationHandler() {}+export function registerUniversalDeprecationHandler() {}+export function unregisterUniversalDeprecationHandler() {}+export function clearBacklog() {}+export function withSilencedDeprecations(_ids, callback) {+ return callback();+}+export async function withSilencedDeprecationsAsync(_ids, callback) {+ return callback();+}+export function isDeprecationSilenced() {+ return false;+}diff --git a/frontend/pretty-text-processor/shims/helpers.js b/frontend/pretty-text-processor/shims/helpers.jsnew file mode 100644index 00000000000..e90c09162a5--- /dev/null+++ b/frontend/pretty-text-processor/shims/helpers.js@@ -0,0 +1,5 @@+import { runtime } from "../runtime-state.js";++export function helperContext() {+ return { siteSettings: { avatar_sizes: runtime.avatarSizes } };+}diff --git a/frontend/pretty-text-processor/shims/i18n.js b/frontend/pretty-text-processor/shims/i18n.jsnew file mode 100644index 00000000000..cf0798d5cfd--- /dev/null+++ b/frontend/pretty-text-processor/shims/i18n.js@@ -0,0 +1,5 @@+/* global __Ruby */+const I18n = { t: (a, b) => __Ruby.t(a, b) };++export default I18n;+export const i18n = I18n.t;diff --git a/lib/asset_processor.rb b/lib/asset_processor.rbindex 07c3909520b..b13c342d1cb 100644--- a/lib/asset_processor.rb+++ b/lib/asset_processor.rb@@ -3,16 +3,20 @@class AssetProcessorBASE_COMPILER_VERSION = 113- PROCESSOR_DIR = "tmp/asset-processor"- LOCK_FILE = "#{PROCESSOR_DIR}/build.lock"-- CACHE_DEPENDENCY_GLOBS = %w[- node_modules/.pnpm/lock.yaml- frontend/asset-processor/**/*.{js,mjs}- frontend/discourse/lib/babel-transform-module-renames.js- frontend/discourse/lib/discourse-source-imports.mjs- frontend/discourse/config/targets.js- ]+ BUNDLE =+ PrecompiledBundle.new(+ dir: "tmp/asset-processor",+ filename_prefix: "asset-processor",+ dependency_globs: %w[+ node_modules/.pnpm/lock.yaml+ frontend/asset-processor/**/*.{js,mjs}+ frontend/discourse/lib/babel-transform-module-renames.js+ frontend/discourse/lib/discourse-source-imports.mjs+ frontend/discourse/config/targets.js+ ],+ ) do+ Discourse::Utils.execute_command("pnpm", "-C=frontend/asset-processor", "node", "build.mjs")+ end@mutex = Mutex.new@ctx_init = Mutex.new@@ -55,64 +59,8 @@ class AssetProcessor@mutexend- def self.build_asset_processor- Discourse::Utils.execute_command("pnpm", "-C=frontend/asset-processor", "node", "build.mjs")- end-- def self.inputs_digest- digest = Digest::MD5.new-- CACHE_DEPENDENCY_GLOBS.each do |pattern|- files = Dir.glob(pattern).sort- raise "No files matched #{pattern}" if files.empty?-- files.each do |file|- digest.update(file)- digest.update(File.read(file))- end- end-- digest.hexdigest.to_i(16).to_s(36) # base36- end-- def self.processor_file_path- "#{PROCESSOR_DIR}/asset-processor-#{inputs_digest}.js"- end-- def self.with_file_lock(&block)- lock_path = "#{Rails.root.join("#{LOCK_FILE}")}"- FileUtils.mkdir_p(File.dirname(lock_path))- File.open(lock_path, File::CREAT | File::RDWR) do |lock_file|- lock_file.flock(File::LOCK_EX)- yield- end- end-- def self.cleanup_old_cache_files- Dir- .glob("#{PROCESSOR_DIR}/asset-processor-*.js")- .reject { it.end_with?(processor_file_path) }- .each { File.delete(it) }- end-def self.load_or_build_processor_source- cache_path = processor_file_path-- if File.exist?(cache_path)- File.read(cache_path)- else- with_file_lock do- if File.exist?(cache_path)- File.read(cache_path)- else- built_source = build_asset_processor- FileUtils.mkdir_p(PROCESSOR_DIR)- File.write(cache_path, built_source)- cleanup_old_cache_files- built_source- end- end- end+ BUNDLE.load_or_buildenddef self.timeoutdiff --git a/lib/precompiled_bundle.rb b/lib/precompiled_bundle.rbnew file mode 100644index 00000000000..418d26b4272--- /dev/null+++ b/lib/precompiled_bundle.rb@@ -0,0 +1,67 @@+# frozen_string_literal: true++# A JS bundle built by an external command and cached on disk, keyed by a digest+# of its inputs. Precompiled during `assets:precompile`; held by composition.+class PrecompiledBundle+ attr_reader :dependency_globs++ def initialize(dir:, filename_prefix:, dependency_globs:, &build)+ @dir = dir+ @filename_prefix = filename_prefix+ @dependency_globs = dependency_globs+ @build = build+ end++ def path+ Rails.root.join(@dir, "#{@filename_prefix}-#{digest}.js")+ end++ def precompiled?+ File.exist?(path)+ end++ def load_or_build+ cache_path = path+ return File.read(cache_path) if File.exist?(cache_path)++ with_lock do+ return File.read(cache_path) if File.exist?(cache_path)+ source = @build.call+ FileUtils.mkdir_p(File.dirname(cache_path))+ Discourse::Utils.atomic_write_file(cache_path, source)+ cleanup_old(cache_path)+ source+ end+ end++ private++ def digest+ digest = Digest::MD5.new+ @dependency_globs.each do |pattern|+ files = Dir.glob(pattern, base: Rails.root).sort+ raise "No files matched #{pattern}" if files.empty?++ files.each do |file|+ digest.update(file)+ digest.update(File.read(Rails.root.join(file)))+ end+ end+ digest.hexdigest.to_i(16).to_s(36)+ end++ def with_lock(&block)+ lock_path = Rails.root.join(@dir, "build.lock")+ FileUtils.mkdir_p(File.dirname(lock_path))+ File.open(lock_path, File::CREAT | File::RDWR) do |lock_file|+ lock_file.flock(File::LOCK_EX)+ yield+ end+ end++ def cleanup_old(keep)+ Dir+ .glob(Rails.root.join(@dir, "#{@filename_prefix}-*.js"))+ .each { |file| File.delete(file) unless file == keep.to_s }+ end+enddiff --git a/lib/pretty_text.rb b/lib/pretty_text.rbindex 8e5153c3277..c112eaa32a3 100644--- a/lib/pretty_text.rb+++ b/lib/pretty_text.rb@@ -38,11 +38,53 @@ module PrettyTextctx.eval(transpiled, filename: module_name)end- def self.ctx_load_directory(ctx:, base_path:, module_prefix:)- Dir["**/*.js", base: base_path].sort.each do |f|- module_name = "#{module_prefix}#{f.delete_suffix(".js")}"- apply_es6_file(ctx: ctx, path: File.join(base_path, f), module_name: module_name)+ # The only modules from the `discourse` package which may be bundled into the+ # server-side renderer. Additions must not transitively depend on+ # browser-only APIs.+ BUNDLED_DISCOURSE_MODULES = %w[+ deprecation-workflow+ lib/avatar-utils+ lib/case-converter+ lib/escape+ lib/get-url+ lib/object+ loader+ static/markdown-it/features+ ]++ CORE_BUNDLE =+ PrecompiledBundle.new(+ dir: "tmp/pretty-text-processor",+ filename_prefix: "pretty-text",+ dependency_globs:+ %w[+ node_modules/.pnpm/lock.yaml+ frontend/pretty-text-processor/**/*.{js,mjs,cjs,json}+ frontend/pretty-text/addon/**/*.js+ frontend/discourse-markdown-it/src/**/*.js+ ] + BUNDLED_DISCOURSE_MODULES.map { "frontend/discourse/app/#{it}.js" },+ ) do+ Discourse::Utils.execute_command(+ "pnpm",+ "-C=frontend/pretty-text-processor",+ "node",+ "build.mjs",+ "--discourse-modules=#{BUNDLED_DISCOURSE_MODULES.join(",")}",+ chdir: Rails.root.to_s,+ )end++ def self.load_or_build_core_bundle+ CORE_BUNDLE.load_or_build+ end++ def self.core_bundle_source+ raise <<~MSG.squish if Rails.env.production? && !CORE_BUNDLE.precompiled?+ PrettyText core bundle has not been precompiled (#{CORE_BUNDLE.path}).+ Run `bin/rake assets:precompile` before booting.+ MSG++ CORE_BUNDLE.load_or_buildenddef self.create_es6_context@@ -82,49 +124,13 @@ module PrettyTextctx.eval("__PRETTY_TEXT = true")+ # `__Ruby` must exist before the bundle runs — it captures it at init.PrettyText::Helpers.instance_methods.each do |method|- ctx.attach("__helpers.#{method}", PrettyText::Helpers.method(method))+ ctx.attach("__Ruby.#{method}", PrettyText::Helpers.method(method))end- root_path = "#{Rails.root.join("frontend")}"- d_node_modules = "#{Rails.root.join("frontend/discourse/node_modules")}"- md_node_modules = "#{Rails.root.join("frontend/discourse-markdown-it/node_modules")}"- ctx.load("#{d_node_modules}/loader.js/dist/loader/loader.js")- ctx.load("#{md_node_modules}/markdown-it/dist/markdown-it.js")- ctx.load("#{md_node_modules}/xss/dist/xss.js")- ctx.load("#{Rails.root.join("lib/pretty_text/vendor-shims.js")}")-- ctx_load_directory(- ctx: ctx,- base_path: "#{root_path}/pretty-text/addon",- module_prefix: "pretty-text/",- )- ctx_load_directory(- ctx: ctx,- base_path: "#{root_path}/discourse-markdown-it/src",- module_prefix: "discourse-markdown-it/",- )-- %w[- discourse/app/deprecation-workflow- discourse/app/lib/get-url- discourse/app/lib/object- discourse/app/lib/deprecated- discourse/app/lib/escape- discourse/app/lib/avatar-utils- discourse/app/lib/case-converter- discourse/app/lib/to-markdown- discourse/app/static/markdown-it/features- ].each do |f|- apply_es6_file(- ctx: ctx,- path: "#{root_path}/#{f}.js",- module_name: f.sub("/addon/", "/").sub("/app/", "/"),- )- end-- ctx.load("#{Rails.root.join("lib/pretty_text/shims.js")}")- ctx.eval("__setUnicode(#{Emoji.unicode_replacements_json})")+ ctx.eval(core_bundle_source, filename: "pretty-text.js")+ ctx.call("__PrettyText.setUnicode", JSON.parse(Emoji.unicode_replacements_json))Discourse.plugins.each do |plugin|Dir@@ -163,7 +169,7 @@ module PrettyTextdef self.reset_translations@mutex.synchronize do- v8.eval("__resetTranslationTree()")+ v8.call("__PrettyText.resetTranslations")v8.low_memory_notification if GlobalSetting.mini_racer_single_threadedendend@@ -195,86 +201,55 @@ module PrettyTextdef self.markdown(text, opts = {})# we use the exact same markdown converter as the client# TODO: use the same extensions on both client and server (in particular the template for mentions)- baked = niltext = text || ""- protect do- context = v8-- custom_emoji = {}- Emoji.custom.map { |e| custom_emoji[e.name] = e.cdn_url }-- allowed_iframes =- DiscoursePluginRegistry.apply_modifier(- :pretty_text_allowed_iframes,- SiteSetting.allowed_iframes.split("|"),- )+ custom_emoji = {}+ Emoji.custom.map { |e| custom_emoji[e.name] = e.cdn_url }- # note, any additional options added to __optInput here must be- # also be added to the buildOptions function in pretty-text.js,- # otherwise they will be discarded- buffer = +<<~JS- __optInput = {};- __optInput.siteSettings = #{SiteSetting.client_settings_json};- __optInput.allowedIframes = #{allowed_iframes.to_json};- #{"__optInput.disableEmojis = true" if opts[:disable_emojis]}- __paths = #{paths_json};- __optInput.getURL = __getURL;- #{"__optInput.features = #{opts[:features].to_json};" if opts[:features]}- #{"__optInput.featuresOverride = #{opts[:features_override].to_json};" if opts[:features_override]}- #{"__optInput.markdownItRules = #{opts[:markdown_it_rules].to_json};" if opts[:markdown_it_rules]}- __optInput.getCurrentUser = __getCurrentUser;- __optInput.lookupAvatar = __lookupAvatar;- __optInput.lookupPrimaryUserGroup = __lookupPrimaryUserGroup;- __optInput.formatUsername = __formatUsername;- __optInput.getTopicInfo = __getTopicInfo;- __optInput.hashtagLookup = __hashtagLookup;- __optInput.customEmoji = #{custom_emoji.to_json};- __optInput.customEmojiTranslation = #{Plugin::CustomEmoji.translations.to_json};- __optInput.emojiUnicodeReplacer = __emojiUnicodeReplacer;- __optInput.emojiDenyList = #{Emoji.denied.to_json};- __optInput.lookupUploadUrls = __lookupUploadUrls;- __optInput.censoredRegexp = #{WordWatcher.serialized_regexps_for_action(:censor).to_json};- __optInput.watchedWordsReplace = #{WordWatcher.regexps_for_action(:replace).to_json};- __optInput.watchedWordsLink = #{WordWatcher.regexps_for_action(:link).to_json};- __optInput.additionalOptions = #{Site.markdown_additional_options.to_json};- __optInput.avatar_sizes = #{SiteSetting.avatar_sizes.to_json};- JS-- buffer << "__optInput.topicId = #{opts[:topic_id].to_i};\n" if opts[:topic_id]- buffer << "__optInput.postId = #{opts[:post_id].to_i};\n" if opts[:post_id]-- if opts[:force_quote_link]- buffer << "__optInput.forceQuoteLink = #{opts[:force_quote_link]};\n"- end-- buffer << "__optInput.userId = #{opts[:user_id].to_i};\n" if opts[:user_id]-- opts[:hashtag_context] = opts[:hashtag_context] || "topic-composer"- hashtag_types_as_js =- HashtagAutocompleteService- .ordered_types_for_context(opts[:hashtag_context])- .map { |t| "'#{t}'" }- .join(",")- buffer << "__optInput.hashtagTypesInPriorityOrder = [#{hashtag_types_as_js}];\n"- buffer << "__optInput.hashtagIcons = #{HashtagAutocompleteService.data_source_icon_map.to_json};\n"-- buffer << "__pluginFeatures = __loadPluginFeatures();"- buffer << "__pt = __DiscourseMarkdownIt.withCustomFeatures(__pluginFeatures).withOptions(__optInput);"-- # Be careful disabling sanitization. We allow for custom emails- buffer << "__pt.disableSanitizer();" if opts[:sanitize] == false+ allowed_iframes =+ DiscoursePluginRegistry.apply_modifier(+ :pretty_text_allowed_iframes,+ SiteSetting.allowed_iframes.split("|"),+ )- opts = context.eval(buffer)+ opts[:hashtag_context] ||= "topic-composer"++ # Any addition here must also be added to buildOptions in discourse-markdown-it.+ opt_input = {+ siteSettings: SiteSetting.client_settings_hash,+ allowedIframes: allowed_iframes,+ paths: paths,+ customEmoji: custom_emoji,+ customEmojiTranslation: Plugin::CustomEmoji.translations,+ emojiDenyList: Emoji.denied,+ censoredRegexp: WordWatcher.serialized_regexps_for_action(:censor),+ watchedWordsReplace: WordWatcher.regexps_for_action(:replace),+ watchedWordsLink: WordWatcher.regexps_for_action(:link),+ additionalOptions: Site.markdown_additional_options,+ avatar_sizes: SiteSetting.avatar_sizes,+ hashtagTypesInPriorityOrder:+ HashtagAutocompleteService.ordered_types_for_context(opts[:hashtag_context]),+ hashtagIcons: HashtagAutocompleteService.data_source_icon_map,+ }++ opt_input[:disableEmojis] = true if opts[:disable_emojis]+ opt_input[:features] = opts[:features] if opts[:features]+ opt_input[:featuresOverride] = opts[:features_override] if opts[:features_override]+ opt_input[:markdownItRules] = opts[:markdown_it_rules] if opts[:markdown_it_rules]+ opt_input[:topicId] = opts[:topic_id].to_i if opts[:topic_id]+ opt_input[:postId] = opts[:post_id].to_i if opts[:post_id]+ opt_input[:forceQuoteLink] = opts[:force_quote_link] if opts[:force_quote_link]+ opt_input[:userId] = opts[:user_id].to_i if opts[:user_id]+ # Be careful disabling sanitization. We allow for custom emails+ opt_input[:disableSanitizer] = true if opts[:sanitize] == false- DiscourseEvent.trigger(:markdown_context, context)- baked = context.eval("__pt.cook(#{text.inspect})")+ protect do+ DiscourseEvent.trigger(:markdown_context, v8)+ v8.call("__PrettyText.cook", text, opt_input)end-- bakedend- def self.paths_json+ def self.pathspaths = { baseUri: Discourse.base_path, CDN: Rails.configuration.action_controller.asset_host }if SiteSetting.Upload.enable_s3_uploads@@ -282,59 +257,48 @@ module PrettyTextpaths[:S3BaseUrl] = Discourse.store.absolute_base_urlend+ paths+ end++ def self.paths_jsonpaths.to_jsonend# leaving this here, cause it invokes v8, don't want to implement twicedef self.avatar_img(avatar_template, size)- protect { v8.eval(<<~JS) }- __optInput = {};- __optInput.avatar_sizes = #{SiteSetting.avatar_sizes.to_json};- __paths = #{paths_json};- require("discourse/lib/avatar-utils").avatarImg({size: #{size.inspect}, avatarTemplate: #{avatar_template.inspect}}, __getURL);- JS+ protect do+ v8.call("__PrettyText.avatarImg", avatar_template, size, paths, SiteSetting.avatar_sizes)+ endenddef self.sanitize(html, opts = {})- protect { v8.eval(<<~JS) }- (() => {- const AllowLister = require("pretty-text/allow-lister").default;- const sanitize = require("pretty-text/sanitizer").sanitize;- return sanitize(#{html.to_s.inspect}, new AllowLister(#{opts.to_json}));- })()- JS+ protect { v8.call("__PrettyText.sanitize", html.to_s, opts) }enddef self.unescape_emoji(title)return title unless SiteSetting.enable_emoji? && title- set = SiteSetting.emoji_set.inspect- custom = Emoji.custom.map { |e| [e.name, e.cdn_url] }.to_h.to_json+ options = {+ paths: paths,+ emojiSet: SiteSetting.emoji_set,+ emojiCDNUrl: SiteSetting.external_emoji_url.presence || "",+ customEmoji: Emoji.custom.map { |e| [e.name, e.cdn_url] }.to_h,+ enableEmojiShortcuts: SiteSetting.enable_emoji_shortcuts,+ inlineEmoji: SiteSetting.enable_inline_emoji_translation,+ }- protect { v8.eval(<<~JS) }- __paths = #{paths_json};- __performEmojiUnescape(#{title.inspect}, {- getURL: __getURL,- emojiSet: #{set},- emojiCDNUrl: "#{SiteSetting.external_emoji_url.presence || ""}",- customEmoji: #{custom},- enableEmojiShortcuts: #{SiteSetting.enable_emoji_shortcuts},- inlineEmoji: #{SiteSetting.enable_inline_emoji_translation}- });- JS+ protect { v8.call("__PrettyText.performEmojiUnescape", title, options) }enddef self.escape_emoji(title)return unless title- replace_emoji_shortcuts = SiteSetting.enable_emoji && SiteSetting.enable_emoji_shortcuts+ options = {+ emojiShortcuts: SiteSetting.enable_emoji && SiteSetting.enable_emoji_shortcuts,+ inlineEmoji: SiteSetting.enable_inline_emoji_translation,+ }- protect { v8.eval(<<~JS) }- __performEmojiEscape(#{title.inspect}, {- emojiShortcuts: #{replace_emoji_shortcuts},- inlineEmoji: #{SiteSetting.enable_inline_emoji_translation}- });- JS+ protect { v8.call("__PrettyText.performEmojiEscape", title, options) }enddef self.cook(raw, opts = {})diff --git a/lib/pretty_text/shims.js b/lib/pretty_text/shims.jsdeleted file mode 100644index 47cfa61810e..00000000000--- a/lib/pretty_text/shims.js+++ /dev/null@@ -1,137 +0,0 @@-/* eslint-disable no-undef, no-unused-vars */-I18n = {- t(a, b) {- return __helpers.t(a, b);- },-};--define("I18n", ["exports"], function (exports) {- exports.default = I18n;-});--define("discourse-i18n", ["exports"], function (exports) {- exports.default = I18n;- exports.i18n = I18n.t;-});--define("discourse/lib/helpers", ["exports"], function (exports) {- exports.helperContext = function () {- return {- siteSettings: { avatar_sizes: __optInput.avatar_sizes },- };- };-});--__emojiUnicodeReplacer = null;--__setUnicode = function (replacements) {- const regexp = new RegExp(__emojiReplacementRegex, "g");-- __emojiUnicodeReplacer = function (text) {- regexp.lastIndex = 0;-- let m;- while ((m = regexp.exec(text)) !== null) {- let match = m[0];-- let replacement = replacements[match];-- if (!replacement) {- // if we can't find replacement for an emoji match- // attempts to look for the same without trailing variation selector- match = match.replace(/\ufe0f$/g, "");- replacement = replacements[match];- }-- if (!replacement) {- continue;- }-- replacement = ":" + replacement + ":";- const before = text.charAt(m.index - 1);- if (!/\B/.test(before)) {- replacement = "\u200b" + replacement;- }- text = text.replace(match, replacement);- }-- // fixes Safari VARIATION SELECTOR-16 issue with some emojis- // https://meta.discourse.org/t/emojis-selected-on-ios-displaying-additional-rectangles/86132- text = text.replace(/\ufe0f/g, "");-- return text;- };-};--__paths = {};--function __getURLNoCDN(url) {- if (!url) {- return url;- }-- // if it's a non relative URL, return it.- if (url !== "/" && !/^\/[^\/]/.test(url)) {- return url;- }-- if (url.startsWith(`${__paths.baseUri}/`) || url === __paths.baseUri) {- return url;- }- if (url[0] !== "/") {- url = "/" + url;- }-- return __paths.baseUri + url;-}--function __getURL(url) {- url = __getURLNoCDN(url);- // only relative urls- if (__paths.CDN && /^\\\/[^\\\/]/.test(url)) {- url = __paths.CDN + url;- } else if (__paths.S3CDN) {- url = url.replace(__paths.S3BaseUrl, __paths.S3CDN);- }- return url;-}--function __lookupUploadUrls(urls) {- return __helpers.lookup_upload_urls(urls);-}--function __getTopicInfo(i) {- return __helpers.get_topic_info(i);-}--function __hashtagLookup(slug, cookingUserId, typesInPriorityOrder) {- return __helpers.hashtag_lookup(slug, cookingUserId, typesInPriorityOrder);-}--function __lookupAvatar(p) {- return require("discourse/lib/avatar-utils").avatarImg(- { size: "tiny", avatarTemplate: __helpers.avatar_template(p) },- __getURL- );-}--function __formatUsername(username) {- return __helpers.format_username(username);-}--function __lookupPrimaryUserGroup(username) {- return __helpers.lookup_primary_user_group(username);-}--function __getCurrentUser(userId) {- return __helpers.get_current_user(userId);-}--__DiscourseMarkdownIt = require("discourse-markdown-it").default;-__buildOptions = require("discourse-markdown-it/options").default;-__performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;-__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex;-__performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;-__resetTranslationTree =- require("discourse-markdown-it/features/emoji").resetTranslationTree;-__loadPluginFeatures = require("discourse/static/markdown-it/features").default;diff --git a/lib/pretty_text/vendor-shims.js b/lib/pretty_text/vendor-shims.jsdeleted file mode 100644index 2b3fc2e8acd..00000000000--- a/lib/pretty_text/vendor-shims.js+++ /dev/null@@ -1,38 +0,0 @@-define("@embroider/macros", ["exports", "require"], function (- __require__,- __exports__-) {- __exports__.importSync = __require__;-});--define("discourse/lib/loader-shim", ["exports", "require"], function (- __exports__,- __require__-) {- __exports__.default = (id, callback) => {- if (!__require__.has(id)) {- define(id, callback);- }- };-});--define("xss", ["exports"], function (__exports__) {- __exports__.default = window.filterXSS;-});--define("markdown-it", ["exports"], function (exports) {- exports.default = window.markdownit;-});--define("@ember/debug", ["exports"], function (exports) {- exports.registerDeprecationHandler = () => {};-});--define("discourse/lib/environment", ["exports"], function (exports) {- exports.isRailsTesting = () => false;-});--define("discourse/lib/source-identifier", ["exports"], function (exports) {- exports.default = () => undefined;- exports.consolePrefix = () => "";-});diff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rakeindex 15671442f6c..3c9ca03d59d 100644--- a/lib/tasks/assets.rake+++ b/lib/tasks/assets.rake@@ -161,8 +161,16 @@ task "assets:precompile:asset_processor": "environment" doAssetProcessor.load_or_build_processor_sourceend+task "assets:precompile:pretty_text": "environment" do+ PrettyText.load_or_build_core_bundle+end+# Run these tasks **before** Rails' "assets:precompile" task-task "assets:precompile": %w[assets:precompile:before assets:precompile:asset_processor]+task "assets:precompile": %w[+ assets:precompile:before+ assets:precompile:asset_processor+ assets:precompile:pretty_text+ ]# Run these tasks **after** Rails' "assets:precompile" taskRake::Task["assets:precompile"].enhance dodiff --git a/lib/tasks/db.rake b/lib/tasks/db.rakeindex 47821aaed06..a7e93bcb163 100644--- a/lib/tasks/db.rake+++ b/lib/tasks/db.rake@@ -150,6 +150,7 @@ task "multisite:migrate" => %w[environmentset_localeassets:precompile:asset_processor+ assets:precompile:pretty_textdb:migrate] do |_, args|DistributedMutex.synchronize(@@ -234,6 +235,7 @@ task "db:migrate" => %w[environmentset_localeassets:precompile:asset_processor+ assets:precompile:pretty_text] do |_, args|DistributedMutex.synchronize("db_migration",diff --git a/lib/tasks/themes.rake b/lib/tasks/themes.rakeindex f4c422a0d26..cf5d20c75b8 100644--- a/lib/tasks/themes.rake+++ b/lib/tasks/themes.rake@@ -326,7 +326,11 @@ def update_themes(version_cache: Concurrent::Map.new)enddesc "Update themes & theme components"-task "themes:update": %w[environment assets:precompile:asset_processor] do+task "themes:update": %w[+ environment+ assets:precompile:asset_processor+ assets:precompile:pretty_text+ ] doif ENV["RAILS_DB"].present?update_themeselsediff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex aa485e1259b..b23e9368dea 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -690,6 +690,27 @@ importers:specifier: ~6.10.0version: 6.10.1(patch_hash=85ff9a16d08431e3bdc9aea8d55280e14f437f1c2a7d8e461de56c614ad48926)(@glimmer/[email protected])([email protected])+ frontend/pretty-text-processor:+ dependencies:+ discourse:+ specifier: workspace:0.0.0+ version: link:../discourse+ discourse-markdown-it:+ specifier: workspace:*+ version: link:../discourse-markdown-it+ markdown-it:+ specifier: ^14.1.1+ version: 14.2.0+ pretty-text:+ specifier: workspace:*+ version: link:../pretty-text+ rolldown:+ specifier: 1.1.2+ version: 1.1.2+ xss:+ specifier: ^1.0.15+ version: 1.0.15+plugins/automation:dependencies:discourse:diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex edcbd306eea..62fd964eccd 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -13,6 +13,7 @@ packages:- "frontend/discourse-i18n"- "frontend/discourse-markdown-it"- "frontend/pretty-text"+- "frontend/pretty-text-processor"- "frontend/asset-processor"- "frontend/discourse-types"- "plugins/automation"diff --git a/spec/lib/asset_processor_spec.rb b/spec/lib/asset_processor_spec.rbindex 313103d2ca8..713cdee9784 100644--- a/spec/lib/asset_processor_spec.rb+++ b/spec/lib/asset_processor_spec.rb@@ -83,7 +83,8 @@ RSpec.describe AssetProcessor do# outside its own directory has to be listed or its changes go unnoticed.processor_dir = File.expand_path("frontend/asset-processor")hashed =- AssetProcessor::CACHE_DEPENDENCY_GLOBS+ AssetProcessor::BUNDLE+ .dependency_globs.flat_map { |glob| Dir.glob(glob) }.map { |path| File.expand_path(path) }.to_set@@ -129,7 +130,7 @@ RSpec.describe AssetProcessor doexpect(hashed).to include(path),"#{Pathname.new(path).relative_path_from(Rails.root)} is imported by " \"#{Pathname.new(imported_by).relative_path_from(Rails.root)} " \- "but is not in CACHE_DEPENDENCY_GLOBS"+ "but is not in AssetProcessor::BUNDLE's dependency_globs"endendenddiff --git a/spec/tasks/assets_precompile_spec.rb b/spec/tasks/assets_precompile_spec.rbindex f2f8ddc2800..895017e1642 100644--- a/spec/tasks/assets_precompile_spec.rb+++ b/spec/tasks/assets_precompile_spec.rb@@ -3,11 +3,10 @@RSpec.describe "assets:precompile" dodescribe "assets:precompile:asset_processor" doit "compiles the js processor" do- FileUtils.rm_rf AssetProcessor::PROCESSOR_DIR+ FileUtils.rm_rf(File.dirname(AssetProcessor::BUNDLE.path))Rake::Task["assets:precompile:asset_processor"].actions.first.call- path = AssetProcessor.processor_file_path- expect(File.exist?(path)).to eq(true)+ expect(File.exist?(AssetProcessor::BUNDLE.path)).to eq(true)endendend