Skip to content

A platform for community discussion. Free, open, simple.

DEV: Build server-side PrettyText bundle with Rolldown (#41997)

David Taylorcfed6b85b6ce
diff --git a/frontend/pretty-text-processor/build.mjs b/frontend/pretty-text-processor/build.mjs
new file mode 100644
index 00000000000..a165fd51fa4
--- /dev/null
+++ b/frontend/pretty-text-processor/build.mjs
@@ -0,0 +1,88 @@
+import { rolldown } from "rolldown";
+import { importGlobPlugin, viteAliasPlugin } from "rolldown/experimental";
+
+const FRONTEND = new URL("../", import.meta.url).pathname;
+const SHIM = (f) => new URL(`./shims/${f}`, import.meta.url).pathname;
+
+// Allowlist of bundleable `discourse` core modules, supplied by
+// PrettyText::BUNDLED_DISCOURSE_MODULES in lib/pretty_text.rb.
+const DISCOURSE_MODULES = process.argv
+ .find((arg) => arg.startsWith("--discourse-modules="))
+ ?.slice("--discourse-modules=".length)
+ .split(",");
+
+if (!DISCOURSE_MODULES?.length) {
+ throw new Error(
+ "Missing --discourse-modules=<comma-separated list> (see lib/pretty_text.rb)"
+ );
+}
+
+const bundle = await rolldown({
+ input: new URL("./entry.js", import.meta.url).pathname,
+ platform: "browser",
+ plugins: [
+ importGlobPlugin(),
+ viteAliasPlugin({
+ entries: [
+ { find: "discourse-i18n", replacement: SHIM("i18n.js") },
+ { find: "discourse/lib/helpers", replacement: SHIM("helpers.js") },
+ {
+ find: "discourse/lib/deprecated",
+ replacement: SHIM("deprecated.js"),
+ },
+ ...DISCOURSE_MODULES.map((m) => ({
+ find: `discourse/${m}`,
+ replacement: `${FRONTEND}discourse/app/${m}.js`,
+ })),
+ {
+ find: /^pretty-text\//,
+ replacement: `${FRONTEND}pretty-text/addon/`,
+ },
+ ],
+ }),
+ {
+ // Exposes the allowlisted modules as a name -> namespace map, so
+ // register-modules.js can define() them without duplicating the list.
+ name: "discourse-modules-map",
+ resolveId(id) {
+ if (id === "virtual:discourse-modules") {
+ return "\0virtual:discourse-modules";
+ }
+ },
+ load(id) {
+ if (id === "\0virtual:discourse-modules") {
+ const modules = DISCOURSE_MODULES.filter((m) => m !== "loader");
+ return [
+ ...modules.map(
+ (m, i) => `import * as mod${i} from "discourse/${m}";`
+ ),
+ "export default {",
+ ...modules.map((m, i) => ` "../discourse/app/${m}.js": mod${i},`),
+ "};",
+ ].join("\n");
+ }
+ },
+ },
+ {
+ // The `discourse` package's exports map would resolve anything under
+ // discourse/*; only imports rewritten by the aliases above are allowed.
+ name: "discourse-import-allowlist",
+ resolveId(id, importer) {
+ if (id.startsWith("discourse/")) {
+ throw new Error(
+ `"${id}" (imported by ${importer}) is not in PrettyText::BUNDLED_DISCOURSE_MODULES`
+ );
+ }
+ },
+ },
+ ],
+ resolve: { extensions: [".js", ".mjs", ".cjs", ".json"] },
+});
+
+const { output } = await bundle.generate({
+ format: "iife",
+ minify: false,
+});
+
+await bundle.close();
+process.stdout.write(output[0].code);
diff --git a/frontend/pretty-text-processor/entry.js b/frontend/pretty-text-processor/entry.js
new file mode 100644
index 00000000000..a29132b686e
--- /dev/null
+++ b/frontend/pretty-text-processor/entry.js
@@ -0,0 +1,6 @@
+import "discourse/loader"; // sets up window.define / window.require — must come first
+import { PrettyTextRubyInterface } from "./pretty-text-ruby-interface.js";
+import { registerCoreModules } from "./register-modules.js";
+
+registerCoreModules();
+globalThis.__PrettyText = PrettyTextRubyInterface;
diff --git a/frontend/pretty-text-processor/package.json b/frontend/pretty-text-processor/package.json
new file mode 100644
index 00000000000..7df081d26b7
--- /dev/null
+++ b/frontend/pretty-text-processor/package.json
@@ -0,0 +1,23 @@
+{
+ "name": "pretty-text-processor",
+ "version": "1.0.0",
+ "private": true,
+ "description": "Uses rolldown to bundle the PrettyText core for loading into mini-racer",
+ "author": "Discourse",
+ "license": "GPL-2.0-only",
+ "keywords": [],
+ "dependencies": {
+ "discourse": "workspace:0.0.0",
+ "discourse-markdown-it": "workspace:*",
+ "markdown-it": "^14.1.1",
+ "pretty-text": "workspace:*",
+ "rolldown": "1.1.2",
+ "xss": "^1.0.15"
+ },
+ "engines": {
+ "node": ">= 20",
+ "npm": "please-use-pnpm",
+ "yarn": "please-use-pnpm",
+ "pnpm": "^10"
+ }
+}
diff --git a/frontend/pretty-text-processor/pretty-text-ruby-interface.js b/frontend/pretty-text-processor/pretty-text-ruby-interface.js
new file mode 100644
index 00000000000..ef53b4cea28
--- /dev/null
+++ b/frontend/pretty-text-processor/pretty-text-ruby-interface.js
@@ -0,0 +1,142 @@
+import AllowLister from "pretty-text/allow-lister";
+import {
+ emojiReplacementRegex,
+ performEmojiEscape,
+ performEmojiUnescape,
+} from "pretty-text/emoji";
+import { sanitize as sanitizeHtml } from "pretty-text/sanitizer";
+import * as avatarUtils from "discourse/lib/avatar-utils";
+import loadPluginFeatures from "discourse/static/markdown-it/features";
+import DiscourseMarkdownIt from "discourse-markdown-it";
+import { resetTranslationTree } from "discourse-markdown-it/features/emoji";
+import { runtime } from "./runtime-state.js";
+
+// Passed to the markdown pipeline as plain callbacks, so functions not methods.
+const ruby = globalThis.__Ruby;
+
+function getURLNoCDN(url) {
+ const paths = runtime.paths;
+ if (!url) {
+ return url;
+ }
+ if (url !== "/" && !/^\/[^\/]/.test(url)) {
+ return url;
+ }
+ if (url.startsWith(`${paths.baseUri}/`) || url === paths.baseUri) {
+ return url;
+ }
+ if (url[0] !== "/") {
+ url = "/" + url;
+ }
+ return paths.baseUri + url;
+}
+
+function getURL(url) {
+ const paths = runtime.paths;
+ url = getURLNoCDN(url);
+ if (paths.CDN && /^\\\/[^\\\/]/.test(url)) {
+ url = paths.CDN + url;
+ } else if (paths.S3CDN) {
+ url = url.replace(paths.S3BaseUrl, paths.S3CDN);
+ }
+ return url;
+}
+
+const lookupUploadUrls = (urls) => ruby.lookup_upload_urls(urls);
+const getTopicInfo = (i) => ruby.get_topic_info(i);
+const hashtagLookup = (slug, cookingUserId, typesInPriorityOrder) =>
+ ruby.hashtag_lookup(slug, cookingUserId, typesInPriorityOrder);
+const lookupAvatar = (p) =>
+ avatarUtils.avatarImg(
+ { size: "tiny", avatarTemplate: ruby.avatar_template(p) },
+ getURL
+ );
+const formatUsername = (username) => ruby.format_username(username);
+const lookupPrimaryUserGroup = (username) =>
+ ruby.lookup_primary_user_group(username);
+const getCurrentUser = (userId) => ruby.get_current_user(userId);
+
+let emojiUnicodeReplacer = null;
+
+function buildEmojiUnicodeReplacer(replacements) {
+ const regexp = new RegExp(emojiReplacementRegex, "g");
+ return function (text) {
+ regexp.lastIndex = 0;
+ let m;
+ while ((m = regexp.exec(text)) !== null) {
+ let match = m[0];
+ let replacement = replacements[match];
+ if (!replacement) {
+ match = match.replace(/️$/g, "");
+ replacement = replacements[match];
+ }
+ if (!replacement) {
+ continue;
+ }
+ replacement = ":" + replacement + ":";
+ const before = text.charAt(m.index - 1);
+ if (!/\B/.test(before)) {
+ replacement = "​" + replacement;
+ }
+ text = text.replace(match, replacement);
+ }
+ text = text.replace(/️/g, "");
+ return text;
+ };
+}
+
+export class PrettyTextRubyInterface {
+ static cook(text, optInput) {
+ runtime.paths = optInput.paths || {};
+ runtime.avatarSizes = optInput.avatar_sizes;
+
+ optInput.getURL = getURL;
+ optInput.getCurrentUser = getCurrentUser;
+ optInput.lookupAvatar = lookupAvatar;
+ optInput.lookupPrimaryUserGroup = lookupPrimaryUserGroup;
+ optInput.formatUsername = formatUsername;
+ optInput.getTopicInfo = getTopicInfo;
+ optInput.hashtagLookup = hashtagLookup;
+ optInput.lookupUploadUrls = lookupUploadUrls;
+ optInput.emojiUnicodeReplacer = emojiUnicodeReplacer;
+
+ const pt =
+ DiscourseMarkdownIt.withCustomFeatures(loadPluginFeatures()).withOptions(
+ optInput
+ );
+
+ if (optInput.disableSanitizer) {
+ pt.disableSanitizer();
+ }
+
+ return pt.cook(text);
+ }
+
+ static sanitize(html, allowListOptions) {
+ return sanitizeHtml(html, new AllowLister(allowListOptions));
+ }
+
+ static avatarImg(avatarTemplate, size, paths, avatarSizes) {
+ runtime.paths = paths || {};
+ runtime.avatarSizes = avatarSizes;
+ return avatarUtils.avatarImg({ size, avatarTemplate }, getURL);
+ }
+
+ static setUnicode(replacements) {
+ emojiUnicodeReplacer = buildEmojiUnicodeReplacer(replacements);
+ }
+
+ static resetTranslations() {
+ resetTranslationTree();
+ }
+
+ static performEmojiUnescape(text, options) {
+ runtime.paths = options.paths || {};
+ options.getURL = getURL;
+ return performEmojiUnescape(text, options);
+ }
+
+ static performEmojiEscape(text, options) {
+ return performEmojiEscape(text, options);
+ }
+}
diff --git a/frontend/pretty-text-processor/register-modules.js b/frontend/pretty-text-processor/register-modules.js
new file mode 100644
index 00000000000..80b5b2c70cc
--- /dev/null
+++ b/frontend/pretty-text-processor/register-modules.js
@@ -0,0 +1,50 @@
+import markdownit from "markdown-it";
+import discourseModules from "virtual:discourse-modules";
+import xss from "xss";
+import * as deprecated from "discourse/lib/deprecated";
+import * as helpersShim from "./shims/helpers.js";
+import * as i18nShim from "./shims/i18n.js";
+
+// oneboxer and upload-short-url are browser-only; they were never usable in
+// the server-side context.
+const ptModules = import.meta.glob(
+ [
+ "../pretty-text/addon/**/*.js",
+ "!../pretty-text/addon/oneboxer.js",
+ "!../pretty-text/addon/upload-short-url.js",
+ ],
+ { eager: true }
+);
+const mdModules = import.meta.glob("../discourse-markdown-it/src/**/*.js", {
+ eager: true,
+});
+
+function defineMultiple(prefix, base, mods) {
+ for (const [key, mod] of Object.entries(mods)) {
+ define(
+ `${prefix}/${key.slice(base.length).replace(/\.js$/, "")}`,
+ () => mod
+ );
+ }
+}
+
+// Registers the core module surface into loader.js so plugins can require() it.
+export function registerCoreModules() {
+ defineMultiple("pretty-text", "../pretty-text/addon/", ptModules);
+ defineMultiple(
+ "discourse-markdown-it",
+ "../discourse-markdown-it/src/",
+ mdModules
+ );
+
+ defineMultiple("discourse", "../discourse/app/", discourseModules);
+ define("discourse/lib/deprecated", () => deprecated);
+
+ globalThis.I18n = i18nShim.default;
+ define("I18n", () => ({ default: i18nShim.default }));
+ define("discourse-i18n", () => i18nShim);
+ define("discourse/lib/helpers", () => helpersShim);
+
+ define("markdown-it", () => ({ default: markdownit }));
+ define("xss", () => ({ default: xss }));
+}
diff --git a/frontend/pretty-text-processor/runtime-state.js b/frontend/pretty-text-processor/runtime-state.js
new file mode 100644
index 00000000000..c2ea4d89038
--- /dev/null
+++ b/frontend/pretty-text-processor/runtime-state.js
@@ -0,0 +1,2 @@
+// Shared with the discourse/lib/helpers shim (reads avatarSizes); set per call.
+export const runtime = { paths: {}, avatarSizes: undefined };
diff --git a/frontend/pretty-text-processor/shims/deprecated.js b/frontend/pretty-text-processor/shims/deprecated.js
new file mode 100644
index 00000000000..29c717ad211
--- /dev/null
+++ b/frontend/pretty-text-processor/shims/deprecated.js
@@ -0,0 +1,30 @@
+// Server-side stand-in for discourse/lib/deprecated; the real one pulls a
+// browser-only dependency chain. Reproduces its console.warn message format.
+export default function deprecated(msg, options = {}) {
+ const { id, since, url } = options;
+ const parts = ["DEPRECATION NOTICE:", msg];
+ if (since) {
+ parts.push(`[deprecated since Discourse ${since}]`);
+ }
+ if (id) {
+ parts.push(`[deprecation id: ${id}]`);
+ }
+ if (url) {
+ parts.push(`[info: ${url}]`);
+ }
+ // eslint-disable-next-line no-console
+ console.warn(parts.join(" "));
+}
+export function registerDeprecationHandler() {}
+export function registerUniversalDeprecationHandler() {}
+export function unregisterUniversalDeprecationHandler() {}
+export function clearBacklog() {}
+export function withSilencedDeprecations(_ids, callback) {
+ return callback();
+}
+export async function withSilencedDeprecationsAsync(_ids, callback) {
+ return callback();
+}
+export function isDeprecationSilenced() {
+ return false;
+}
diff --git a/frontend/pretty-text-processor/shims/helpers.js b/frontend/pretty-text-processor/shims/helpers.js
new file mode 100644
index 00000000000..e90c09162a5
--- /dev/null
+++ b/frontend/pretty-text-processor/shims/helpers.js
@@ -0,0 +1,5 @@
+import { runtime } from "../runtime-state.js";
+
+export function helperContext() {
+ return { siteSettings: { avatar_sizes: runtime.avatarSizes } };
+}
diff --git a/frontend/pretty-text-processor/shims/i18n.js b/frontend/pretty-text-processor/shims/i18n.js
new file mode 100644
index 00000000000..cf0798d5cfd
--- /dev/null
+++ b/frontend/pretty-text-processor/shims/i18n.js
@@ -0,0 +1,5 @@
+/* global __Ruby */
+const I18n = { t: (a, b) => __Ruby.t(a, b) };
+
+export default I18n;
+export const i18n = I18n.t;
diff --git a/lib/asset_processor.rb b/lib/asset_processor.rb
index 07c3909520b..b13c342d1cb 100644
--- a/lib/asset_processor.rb
+++ b/lib/asset_processor.rb
@@ -3,16 +3,20 @@
class AssetProcessor
BASE_COMPILER_VERSION = 113
- PROCESSOR_DIR = "tmp/asset-processor"
- LOCK_FILE = "#{PROCESSOR_DIR}/build.lock"
-
- CACHE_DEPENDENCY_GLOBS = %w[
- node_modules/.pnpm/lock.yaml
- frontend/asset-processor/**/*.{js,mjs}
- frontend/discourse/lib/babel-transform-module-renames.js
- frontend/discourse/lib/discourse-source-imports.mjs
- frontend/discourse/config/targets.js
- ]
+ BUNDLE =
+ PrecompiledBundle.new(
+ dir: "tmp/asset-processor",
+ filename_prefix: "asset-processor",
+ dependency_globs: %w[
+ node_modules/.pnpm/lock.yaml
+ frontend/asset-processor/**/*.{js,mjs}
+ frontend/discourse/lib/babel-transform-module-renames.js
+ frontend/discourse/lib/discourse-source-imports.mjs
+ frontend/discourse/config/targets.js
+ ],
+ ) do
+ Discourse::Utils.execute_command("pnpm", "-C=frontend/asset-processor", "node", "build.mjs")
+ end
@mutex = Mutex.new
@ctx_init = Mutex.new
@@ -55,64 +59,8 @@ class AssetProcessor
@mutex
end
- def self.build_asset_processor
- Discourse::Utils.execute_command("pnpm", "-C=frontend/asset-processor", "node", "build.mjs")
- end
-
- def self.inputs_digest
- digest = Digest::MD5.new
-
- CACHE_DEPENDENCY_GLOBS.each do |pattern|
- files = Dir.glob(pattern).sort
- raise "No files matched #{pattern}" if files.empty?
-
- files.each do |file|
- digest.update(file)
- digest.update(File.read(file))
- end
- end
-
- digest.hexdigest.to_i(16).to_s(36) # base36
- end
-
- def self.processor_file_path
- "#{PROCESSOR_DIR}/asset-processor-#{inputs_digest}.js"
- end
-
- def self.with_file_lock(&block)
- lock_path = "#{Rails.root.join("#{LOCK_FILE}")}"
- FileUtils.mkdir_p(File.dirname(lock_path))
- File.open(lock_path, File::CREAT | File::RDWR) do |lock_file|
- lock_file.flock(File::LOCK_EX)
- yield
- end
- end
-
- def self.cleanup_old_cache_files
- Dir
- .glob("#{PROCESSOR_DIR}/asset-processor-*.js")
- .reject { it.end_with?(processor_file_path) }
- .each { File.delete(it) }
- end
-
def self.load_or_build_processor_source
- cache_path = processor_file_path
-
- if File.exist?(cache_path)
- File.read(cache_path)
- else
- with_file_lock do
- if File.exist?(cache_path)
- File.read(cache_path)
- else
- built_source = build_asset_processor
- FileUtils.mkdir_p(PROCESSOR_DIR)
- File.write(cache_path, built_source)
- cleanup_old_cache_files
- built_source
- end
- end
- end
+ BUNDLE.load_or_build
end
def self.timeout
diff --git a/lib/precompiled_bundle.rb b/lib/precompiled_bundle.rb
new file mode 100644
index 00000000000..418d26b4272
--- /dev/null
+++ b/lib/precompiled_bundle.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+# A JS bundle built by an external command and cached on disk, keyed by a digest
+# of its inputs. Precompiled during `assets:precompile`; held by composition.
+class PrecompiledBundle
+ attr_reader :dependency_globs
+
+ def initialize(dir:, filename_prefix:, dependency_globs:, &build)
+ @dir = dir
+ @filename_prefix = filename_prefix
+ @dependency_globs = dependency_globs
+ @build = build
+ end
+
+ def path
+ Rails.root.join(@dir, "#{@filename_prefix}-#{digest}.js")
+ end
+
+ def precompiled?
+ File.exist?(path)
+ end
+
+ def load_or_build
+ cache_path = path
+ return File.read(cache_path) if File.exist?(cache_path)
+
+ with_lock do
+ return File.read(cache_path) if File.exist?(cache_path)
+ source = @build.call
+ FileUtils.mkdir_p(File.dirname(cache_path))
+ Discourse::Utils.atomic_write_file(cache_path, source)
+ cleanup_old(cache_path)
+ source
+ end
+ end
+
+ private
+
+ def digest
+ digest = Digest::MD5.new
+ @dependency_globs.each do |pattern|
+ files = Dir.glob(pattern, base: Rails.root).sort
+ raise "No files matched #{pattern}" if files.empty?
+
+ files.each do |file|
+ digest.update(file)
+ digest.update(File.read(Rails.root.join(file)))
+ end
+ end
+ digest.hexdigest.to_i(16).to_s(36)
+ end
+
+ def with_lock(&block)
+ lock_path = Rails.root.join(@dir, "build.lock")
+ FileUtils.mkdir_p(File.dirname(lock_path))
+ File.open(lock_path, File::CREAT | File::RDWR) do |lock_file|
+ lock_file.flock(File::LOCK_EX)
+ yield
+ end
+ end
+
+ def cleanup_old(keep)
+ Dir
+ .glob(Rails.root.join(@dir, "#{@filename_prefix}-*.js"))
+ .each { |file| File.delete(file) unless file == keep.to_s }
+ end
+end
diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb
index 8e5153c3277..c112eaa32a3 100644
--- a/lib/pretty_text.rb
+++ b/lib/pretty_text.rb
@@ -38,11 +38,53 @@ module PrettyText
ctx.eval(transpiled, filename: module_name)
end
- def self.ctx_load_directory(ctx:, base_path:, module_prefix:)
- Dir["**/*.js", base: base_path].sort.each do |f|
- module_name = "#{module_prefix}#{f.delete_suffix(".js")}"
- apply_es6_file(ctx: ctx, path: File.join(base_path, f), module_name: module_name)
+ # The only modules from the `discourse` package which may be bundled into the
+ # server-side renderer. Additions must not transitively depend on
+ # browser-only APIs.
+ BUNDLED_DISCOURSE_MODULES = %w[
+ deprecation-workflow
+ lib/avatar-utils
+ lib/case-converter
+ lib/escape
+ lib/get-url
+ lib/object
+ loader
+ static/markdown-it/features
+ ]
+
+ CORE_BUNDLE =
+ PrecompiledBundle.new(
+ dir: "tmp/pretty-text-processor",
+ filename_prefix: "pretty-text",
+ dependency_globs:
+ %w[
+ node_modules/.pnpm/lock.yaml
+ frontend/pretty-text-processor/**/*.{js,mjs,cjs,json}
+ frontend/pretty-text/addon/**/*.js
+ frontend/discourse-markdown-it/src/**/*.js
+ ] + BUNDLED_DISCOURSE_MODULES.map { "frontend/discourse/app/#{it}.js" },
+ ) do
+ Discourse::Utils.execute_command(
+ "pnpm",
+ "-C=frontend/pretty-text-processor",
+ "node",
+ "build.mjs",
+ "--discourse-modules=#{BUNDLED_DISCOURSE_MODULES.join(",")}",
+ chdir: Rails.root.to_s,
+ )
end
+
+ def self.load_or_build_core_bundle
+ CORE_BUNDLE.load_or_build
+ end
+
+ def self.core_bundle_source
+ raise <<~MSG.squish if Rails.env.production? && !CORE_BUNDLE.precompiled?
+ PrettyText core bundle has not been precompiled (#{CORE_BUNDLE.path}).
+ Run `bin/rake assets:precompile` before booting.
+ MSG
+
+ CORE_BUNDLE.load_or_build
end
def self.create_es6_context
@@ -82,49 +124,13 @@ module PrettyText
ctx.eval("__PRETTY_TEXT = true")
+ # `__Ruby` must exist before the bundle runs — it captures it at init.
PrettyText::Helpers.instance_methods.each do |method|
- ctx.attach("__helpers.#{method}", PrettyText::Helpers.method(method))
+ ctx.attach("__Ruby.#{method}", PrettyText::Helpers.method(method))
end
- root_path = "#{Rails.root.join("frontend")}"
- d_node_modules = "#{Rails.root.join("frontend/discourse/node_modules")}"
- md_node_modules = "#{Rails.root.join("frontend/discourse-markdown-it/node_modules")}"
- ctx.load("#{d_node_modules}/loader.js/dist/loader/loader.js")
- ctx.load("#{md_node_modules}/markdown-it/dist/markdown-it.js")
- ctx.load("#{md_node_modules}/xss/dist/xss.js")
- ctx.load("#{Rails.root.join("lib/pretty_text/vendor-shims.js")}")
-
- ctx_load_directory(
- ctx: ctx,
- base_path: "#{root_path}/pretty-text/addon",
- module_prefix: "pretty-text/",
- )
- ctx_load_directory(
- ctx: ctx,
- base_path: "#{root_path}/discourse-markdown-it/src",
- module_prefix: "discourse-markdown-it/",
- )
-
- %w[
- discourse/app/deprecation-workflow
- discourse/app/lib/get-url
- discourse/app/lib/object
- discourse/app/lib/deprecated
- discourse/app/lib/escape
- discourse/app/lib/avatar-utils
- discourse/app/lib/case-converter
- discourse/app/lib/to-markdown
- discourse/app/static/markdown-it/features
- ].each do |f|
- apply_es6_file(
- ctx: ctx,
- path: "#{root_path}/#{f}.js",
- module_name: f.sub("/addon/", "/").sub("/app/", "/"),
- )
- end
-
- ctx.load("#{Rails.root.join("lib/pretty_text/shims.js")}")
- ctx.eval("__setUnicode(#{Emoji.unicode_replacements_json})")
+ ctx.eval(core_bundle_source, filename: "pretty-text.js")
+ ctx.call("__PrettyText.setUnicode", JSON.parse(Emoji.unicode_replacements_json))
Discourse.plugins.each do |plugin|
Dir
@@ -163,7 +169,7 @@ module PrettyText
def self.reset_translations
@mutex.synchronize do
- v8.eval("__resetTranslationTree()")
+ v8.call("__PrettyText.resetTranslations")
v8.low_memory_notification if GlobalSetting.mini_racer_single_threaded
end
end
@@ -195,86 +201,55 @@ module PrettyText
def self.markdown(text, opts = {})
# we use the exact same markdown converter as the client
# TODO: use the same extensions on both client and server (in particular the template for mentions)
- baked = nil
text = text || ""
- protect do
- context = v8
-
- custom_emoji = {}
- Emoji.custom.map { |e| custom_emoji[e.name] = e.cdn_url }
-
- allowed_iframes =
- DiscoursePluginRegistry.apply_modifier(
- :pretty_text_allowed_iframes,
- SiteSetting.allowed_iframes.split("|"),
- )
+ custom_emoji = {}
+ Emoji.custom.map { |e| custom_emoji[e.name] = e.cdn_url }
- # note, any additional options added to __optInput here must be
- # also be added to the buildOptions function in pretty-text.js,
- # otherwise they will be discarded
- buffer = +<<~JS
- __optInput = {};
- __optInput.siteSettings = #{SiteSetting.client_settings_json};
- __optInput.allowedIframes = #{allowed_iframes.to_json};
- #{"__optInput.disableEmojis = true" if opts[:disable_emojis]}
- __paths = #{paths_json};
- __optInput.getURL = __getURL;
- #{"__optInput.features = #{opts[:features].to_json};" if opts[:features]}
- #{"__optInput.featuresOverride = #{opts[:features_override].to_json};" if opts[:features_override]}
- #{"__optInput.markdownItRules = #{opts[:markdown_it_rules].to_json};" if opts[:markdown_it_rules]}
- __optInput.getCurrentUser = __getCurrentUser;
- __optInput.lookupAvatar = __lookupAvatar;
- __optInput.lookupPrimaryUserGroup = __lookupPrimaryUserGroup;
- __optInput.formatUsername = __formatUsername;
- __optInput.getTopicInfo = __getTopicInfo;
- __optInput.hashtagLookup = __hashtagLookup;
- __optInput.customEmoji = #{custom_emoji.to_json};
- __optInput.customEmojiTranslation = #{Plugin::CustomEmoji.translations.to_json};
- __optInput.emojiUnicodeReplacer = __emojiUnicodeReplacer;
- __optInput.emojiDenyList = #{Emoji.denied.to_json};
- __optInput.lookupUploadUrls = __lookupUploadUrls;
- __optInput.censoredRegexp = #{WordWatcher.serialized_regexps_for_action(:censor).to_json};
- __optInput.watchedWordsReplace = #{WordWatcher.regexps_for_action(:replace).to_json};
- __optInput.watchedWordsLink = #{WordWatcher.regexps_for_action(:link).to_json};
- __optInput.additionalOptions = #{Site.markdown_additional_options.to_json};
- __optInput.avatar_sizes = #{SiteSetting.avatar_sizes.to_json};
- JS
-
- buffer << "__optInput.topicId = #{opts[:topic_id].to_i};\n" if opts[:topic_id]
- buffer << "__optInput.postId = #{opts[:post_id].to_i};\n" if opts[:post_id]
-
- if opts[:force_quote_link]
- buffer << "__optInput.forceQuoteLink = #{opts[:force_quote_link]};\n"
- end
-
- buffer << "__optInput.userId = #{opts[:user_id].to_i};\n" if opts[:user_id]
-
- opts[:hashtag_context] = opts[:hashtag_context] || "topic-composer"
- hashtag_types_as_js =
- HashtagAutocompleteService
- .ordered_types_for_context(opts[:hashtag_context])
- .map { |t| "'#{t}'" }
- .join(",")
- buffer << "__optInput.hashtagTypesInPriorityOrder = [#{hashtag_types_as_js}];\n"
- buffer << "__optInput.hashtagIcons = #{HashtagAutocompleteService.data_source_icon_map.to_json};\n"
-
- buffer << "__pluginFeatures = __loadPluginFeatures();"
- buffer << "__pt = __DiscourseMarkdownIt.withCustomFeatures(__pluginFeatures).withOptions(__optInput);"
-
- # Be careful disabling sanitization. We allow for custom emails
- buffer << "__pt.disableSanitizer();" if opts[:sanitize] == false
+ allowed_iframes =
+ DiscoursePluginRegistry.apply_modifier(
+ :pretty_text_allowed_iframes,
+ SiteSetting.allowed_iframes.split("|"),
+ )
- opts = context.eval(buffer)
+ opts[:hashtag_context] ||= "topic-composer"
+
+ # Any addition here must also be added to buildOptions in discourse-markdown-it.
+ opt_input = {
+ siteSettings: SiteSetting.client_settings_hash,
+ allowedIframes: allowed_iframes,
+ paths: paths,
+ customEmoji: custom_emoji,
+ customEmojiTranslation: Plugin::CustomEmoji.translations,
+ emojiDenyList: Emoji.denied,
+ censoredRegexp: WordWatcher.serialized_regexps_for_action(:censor),
+ watchedWordsReplace: WordWatcher.regexps_for_action(:replace),
+ watchedWordsLink: WordWatcher.regexps_for_action(:link),
+ additionalOptions: Site.markdown_additional_options,
+ avatar_sizes: SiteSetting.avatar_sizes,
+ hashtagTypesInPriorityOrder:
+ HashtagAutocompleteService.ordered_types_for_context(opts[:hashtag_context]),
+ hashtagIcons: HashtagAutocompleteService.data_source_icon_map,
+ }
+
+ opt_input[:disableEmojis] = true if opts[:disable_emojis]
+ opt_input[:features] = opts[:features] if opts[:features]
+ opt_input[:featuresOverride] = opts[:features_override] if opts[:features_override]
+ opt_input[:markdownItRules] = opts[:markdown_it_rules] if opts[:markdown_it_rules]
+ opt_input[:topicId] = opts[:topic_id].to_i if opts[:topic_id]
+ opt_input[:postId] = opts[:post_id].to_i if opts[:post_id]
+ opt_input[:forceQuoteLink] = opts[:force_quote_link] if opts[:force_quote_link]
+ opt_input[:userId] = opts[:user_id].to_i if opts[:user_id]
+ # Be careful disabling sanitization. We allow for custom emails
+ opt_input[:disableSanitizer] = true if opts[:sanitize] == false
- DiscourseEvent.trigger(:markdown_context, context)
- baked = context.eval("__pt.cook(#{text.inspect})")
+ protect do
+ DiscourseEvent.trigger(:markdown_context, v8)
+ v8.call("__PrettyText.cook", text, opt_input)
end
-
- baked
end
- def self.paths_json
+ def self.paths
paths = { baseUri: Discourse.base_path, CDN: Rails.configuration.action_controller.asset_host }
if SiteSetting.Upload.enable_s3_uploads
@@ -282,59 +257,48 @@ module PrettyText
paths[:S3BaseUrl] = Discourse.store.absolute_base_url
end
+ paths
+ end
+
+ def self.paths_json
paths.to_json
end
# leaving this here, cause it invokes v8, don't want to implement twice
def self.avatar_img(avatar_template, size)
- protect { v8.eval(<<~JS) }
- __optInput = {};
- __optInput.avatar_sizes = #{SiteSetting.avatar_sizes.to_json};
- __paths = #{paths_json};
- require("discourse/lib/avatar-utils").avatarImg({size: #{size.inspect}, avatarTemplate: #{avatar_template.inspect}}, __getURL);
- JS
+ protect do
+ v8.call("__PrettyText.avatarImg", avatar_template, size, paths, SiteSetting.avatar_sizes)
+ end
end
def self.sanitize(html, opts = {})
- protect { v8.eval(<<~JS) }
- (() => {
- const AllowLister = require("pretty-text/allow-lister").default;
- const sanitize = require("pretty-text/sanitizer").sanitize;
- return sanitize(#{html.to_s.inspect}, new AllowLister(#{opts.to_json}));
- })()
- JS
+ protect { v8.call("__PrettyText.sanitize", html.to_s, opts) }
end
def self.unescape_emoji(title)
return title unless SiteSetting.enable_emoji? && title
- set = SiteSetting.emoji_set.inspect
- custom = Emoji.custom.map { |e| [e.name, e.cdn_url] }.to_h.to_json
+ options = {
+ paths: paths,
+ emojiSet: SiteSetting.emoji_set,
+ emojiCDNUrl: SiteSetting.external_emoji_url.presence || "",
+ customEmoji: Emoji.custom.map { |e| [e.name, e.cdn_url] }.to_h,
+ enableEmojiShortcuts: SiteSetting.enable_emoji_shortcuts,
+ inlineEmoji: SiteSetting.enable_inline_emoji_translation,
+ }
- protect { v8.eval(<<~JS) }
- __paths = #{paths_json};
- __performEmojiUnescape(#{title.inspect}, {
- getURL: __getURL,
- emojiSet: #{set},
- emojiCDNUrl: "#{SiteSetting.external_emoji_url.presence || ""}",
- customEmoji: #{custom},
- enableEmojiShortcuts: #{SiteSetting.enable_emoji_shortcuts},
- inlineEmoji: #{SiteSetting.enable_inline_emoji_translation}
- });
- JS
+ protect { v8.call("__PrettyText.performEmojiUnescape", title, options) }
end
def self.escape_emoji(title)
return unless title
- replace_emoji_shortcuts = SiteSetting.enable_emoji && SiteSetting.enable_emoji_shortcuts
+ options = {
+ emojiShortcuts: SiteSetting.enable_emoji && SiteSetting.enable_emoji_shortcuts,
+ inlineEmoji: SiteSetting.enable_inline_emoji_translation,
+ }
- protect { v8.eval(<<~JS) }
- __performEmojiEscape(#{title.inspect}, {
- emojiShortcuts: #{replace_emoji_shortcuts},
- inlineEmoji: #{SiteSetting.enable_inline_emoji_translation}
- });
- JS
+ protect { v8.call("__PrettyText.performEmojiEscape", title, options) }
end
def self.cook(raw, opts = {})
diff --git a/lib/pretty_text/shims.js b/lib/pretty_text/shims.js
deleted file mode 100644
index 47cfa61810e..00000000000
--- a/lib/pretty_text/shims.js
+++ /dev/null
@@ -1,137 +0,0 @@
-/* eslint-disable no-undef, no-unused-vars */
-I18n = {
- t(a, b) {
- return __helpers.t(a, b);
- },
-};
-
-define("I18n", ["exports"], function (exports) {
- exports.default = I18n;
-});
-
-define("discourse-i18n", ["exports"], function (exports) {
- exports.default = I18n;
- exports.i18n = I18n.t;
-});
-
-define("discourse/lib/helpers", ["exports"], function (exports) {
- exports.helperContext = function () {
- return {
- siteSettings: { avatar_sizes: __optInput.avatar_sizes },
- };
- };
-});
-
-__emojiUnicodeReplacer = null;
-
-__setUnicode = function (replacements) {
- const regexp = new RegExp(__emojiReplacementRegex, "g");
-
- __emojiUnicodeReplacer = function (text) {
- regexp.lastIndex = 0;
-
- let m;
- while ((m = regexp.exec(text)) !== null) {
- let match = m[0];
-
- let replacement = replacements[match];
-
- if (!replacement) {
- // if we can't find replacement for an emoji match
- // attempts to look for the same without trailing variation selector
- match = match.replace(/\ufe0f$/g, "");
- replacement = replacements[match];
- }
-
- if (!replacement) {
- continue;
- }
-
- replacement = ":" + replacement + ":";
- const before = text.charAt(m.index - 1);
- if (!/\B/.test(before)) {
- replacement = "\u200b" + replacement;
- }
- text = text.replace(match, replacement);
- }
-
- // fixes Safari VARIATION SELECTOR-16 issue with some emojis
- // https://meta.discourse.org/t/emojis-selected-on-ios-displaying-additional-rectangles/86132
- text = text.replace(/\ufe0f/g, "");
-
- return text;
- };
-};
-
-__paths = {};
-
-function __getURLNoCDN(url) {
- if (!url) {
- return url;
- }
-
- // if it's a non relative URL, return it.
- if (url !== "/" && !/^\/[^\/]/.test(url)) {
- return url;
- }
-
- if (url.startsWith(`${__paths.baseUri}/`) || url === __paths.baseUri) {
- return url;
- }
- if (url[0] !== "/") {
- url = "/" + url;
- }
-
- return __paths.baseUri + url;
-}
-
-function __getURL(url) {
- url = __getURLNoCDN(url);
- // only relative urls
- if (__paths.CDN && /^\\\/[^\\\/]/.test(url)) {
- url = __paths.CDN + url;
- } else if (__paths.S3CDN) {
- url = url.replace(__paths.S3BaseUrl, __paths.S3CDN);
- }
- return url;
-}
-
-function __lookupUploadUrls(urls) {
- return __helpers.lookup_upload_urls(urls);
-}
-
-function __getTopicInfo(i) {
- return __helpers.get_topic_info(i);
-}
-
-function __hashtagLookup(slug, cookingUserId, typesInPriorityOrder) {
- return __helpers.hashtag_lookup(slug, cookingUserId, typesInPriorityOrder);
-}
-
-function __lookupAvatar(p) {
- return require("discourse/lib/avatar-utils").avatarImg(
- { size: "tiny", avatarTemplate: __helpers.avatar_template(p) },
- __getURL
- );
-}
-
-function __formatUsername(username) {
- return __helpers.format_username(username);
-}
-
-function __lookupPrimaryUserGroup(username) {
- return __helpers.lookup_primary_user_group(username);
-}
-
-function __getCurrentUser(userId) {
- return __helpers.get_current_user(userId);
-}
-
-__DiscourseMarkdownIt = require("discourse-markdown-it").default;
-__buildOptions = require("discourse-markdown-it/options").default;
-__performEmojiUnescape = require("pretty-text/emoji").performEmojiUnescape;
-__emojiReplacementRegex = require("pretty-text/emoji").emojiReplacementRegex;
-__performEmojiEscape = require("pretty-text/emoji").performEmojiEscape;
-__resetTranslationTree =
- require("discourse-markdown-it/features/emoji").resetTranslationTree;
-__loadPluginFeatures = require("discourse/static/markdown-it/features").default;
diff --git a/lib/pretty_text/vendor-shims.js b/lib/pretty_text/vendor-shims.js
deleted file mode 100644
index 2b3fc2e8acd..00000000000
--- a/lib/pretty_text/vendor-shims.js
+++ /dev/null
@@ -1,38 +0,0 @@
-define("@embroider/macros", ["exports", "require"], function (
- __require__,
- __exports__
-) {
- __exports__.importSync = __require__;
-});
-
-define("discourse/lib/loader-shim", ["exports", "require"], function (
- __exports__,
- __require__
-) {
- __exports__.default = (id, callback) => {
- if (!__require__.has(id)) {
- define(id, callback);
- }
- };
-});
-
-define("xss", ["exports"], function (__exports__) {
- __exports__.default = window.filterXSS;
-});
-
-define("markdown-it", ["exports"], function (exports) {
- exports.default = window.markdownit;
-});
-
-define("@ember/debug", ["exports"], function (exports) {
- exports.registerDeprecationHandler = () => {};
-});
-
-define("discourse/lib/environment", ["exports"], function (exports) {
- exports.isRailsTesting = () => false;
-});
-
-define("discourse/lib/source-identifier", ["exports"], function (exports) {
- exports.default = () => undefined;
- exports.consolePrefix = () => "";
-});
diff --git a/lib/tasks/assets.rake b/lib/tasks/assets.rake
index 15671442f6c..3c9ca03d59d 100644
--- a/lib/tasks/assets.rake
+++ b/lib/tasks/assets.rake
@@ -161,8 +161,16 @@ task "assets:precompile:asset_processor": "environment" do
AssetProcessor.load_or_build_processor_source
end
+task "assets:precompile:pretty_text": "environment" do
+ PrettyText.load_or_build_core_bundle
+end
+
# Run these tasks **before** Rails' "assets:precompile" task
-task "assets:precompile": %w[assets:precompile:before assets:precompile:asset_processor]
+task "assets:precompile": %w[
+ assets:precompile:before
+ assets:precompile:asset_processor
+ assets:precompile:pretty_text
+ ]
# Run these tasks **after** Rails' "assets:precompile" task
Rake::Task["assets:precompile"].enhance do
diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake
index 47821aaed06..a7e93bcb163 100644
--- a/lib/tasks/db.rake
+++ b/lib/tasks/db.rake
@@ -150,6 +150,7 @@ task "multisite:migrate" => %w[
environment
set_locale
assets:precompile:asset_processor
+ assets:precompile:pretty_text
db:migrate
] do |_, args|
DistributedMutex.synchronize(
@@ -234,6 +235,7 @@ task "db:migrate" => %w[
environment
set_locale
assets:precompile:asset_processor
+ assets:precompile:pretty_text
] do |_, args|
DistributedMutex.synchronize(
"db_migration",
diff --git a/lib/tasks/themes.rake b/lib/tasks/themes.rake
index f4c422a0d26..cf5d20c75b8 100644
--- a/lib/tasks/themes.rake
+++ b/lib/tasks/themes.rake
@@ -326,7 +326,11 @@ def update_themes(version_cache: Concurrent::Map.new)
end
desc "Update themes & theme components"
-task "themes:update": %w[environment assets:precompile:asset_processor] do
+task "themes:update": %w[
+ environment
+ assets:precompile:asset_processor
+ assets:precompile:pretty_text
+ ] do
if ENV["RAILS_DB"].present?
update_themes
else
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index aa485e1259b..b23e9368dea 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -690,6 +690,27 @@ importers:
specifier: ~6.10.0
version: 6.10.1(patch_hash=85ff9a16d08431e3bdc9aea8d55280e14f437f1c2a7d8e461de56c614ad48926)(@glimmer/[email protected])([email protected])
+ frontend/pretty-text-processor:
+ dependencies:
+ discourse:
+ specifier: workspace:0.0.0
+ version: link:../discourse
+ discourse-markdown-it:
+ specifier: workspace:*
+ version: link:../discourse-markdown-it
+ markdown-it:
+ specifier: ^14.1.1
+ version: 14.2.0
+ pretty-text:
+ specifier: workspace:*
+ version: link:../pretty-text
+ rolldown:
+ specifier: 1.1.2
+ version: 1.1.2
+ xss:
+ specifier: ^1.0.15
+ version: 1.0.15
+
plugins/automation:
dependencies:
discourse:
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index edcbd306eea..62fd964eccd 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -13,6 +13,7 @@ packages:
- "frontend/discourse-i18n"
- "frontend/discourse-markdown-it"
- "frontend/pretty-text"
+- "frontend/pretty-text-processor"
- "frontend/asset-processor"
- "frontend/discourse-types"
- "plugins/automation"
diff --git a/spec/lib/asset_processor_spec.rb b/spec/lib/asset_processor_spec.rb
index 313103d2ca8..713cdee9784 100644
--- a/spec/lib/asset_processor_spec.rb
+++ b/spec/lib/asset_processor_spec.rb
@@ -83,7 +83,8 @@ RSpec.describe AssetProcessor do
# outside its own directory has to be listed or its changes go unnoticed.
processor_dir = File.expand_path("frontend/asset-processor")
hashed =
- AssetProcessor::CACHE_DEPENDENCY_GLOBS
+ AssetProcessor::BUNDLE
+ .dependency_globs
.flat_map { |glob| Dir.glob(glob) }
.map { |path| File.expand_path(path) }
.to_set
@@ -129,7 +130,7 @@ RSpec.describe AssetProcessor do
expect(hashed).to include(path),
"#{Pathname.new(path).relative_path_from(Rails.root)} is imported by " \
"#{Pathname.new(imported_by).relative_path_from(Rails.root)} " \
- "but is not in CACHE_DEPENDENCY_GLOBS"
+ "but is not in AssetProcessor::BUNDLE's dependency_globs"
end
end
end
diff --git a/spec/tasks/assets_precompile_spec.rb b/spec/tasks/assets_precompile_spec.rb
index f2f8ddc2800..895017e1642 100644
--- a/spec/tasks/assets_precompile_spec.rb
+++ b/spec/tasks/assets_precompile_spec.rb
@@ -3,11 +3,10 @@
RSpec.describe "assets:precompile" do
describe "assets:precompile:asset_processor" do
it "compiles the js processor" do
- FileUtils.rm_rf AssetProcessor::PROCESSOR_DIR
+ FileUtils.rm_rf(File.dirname(AssetProcessor::BUNDLE.path))
Rake::Task["assets:precompile:asset_processor"].actions.first.call
- path = AssetProcessor.processor_file_path
- expect(File.exist?(path)).to eq(true)
+ expect(File.exist?(AssetProcessor::BUNDLE.path)).to eq(true)
end
end
end