A platform for community discussion. Free, open, simple.
FEATURE: Add native Markdown endpoints for topics and lists (#43843)
Keegan George2b08f13114fd
diff --git a/.licensed.yml b/.licensed.ymlindex cd438a25ac9..1c5f3916f55 100644--- a/.licensed.yml+++ b/.licensed.yml@@ -81,6 +81,7 @@ reviewed:- r2 # Apache-2.0 (Twitter)- raindrops # LGPL-2.1+- rbs # BSD-2-Clause or Ruby+ - reverse_markdown # wtfpl accepted for this dependency- rubyzip # Ruby- securerandom # Ruby- sidekiq # LGPL (Sidekiq)diff --git a/Gemfile b/Gemfileindex 68d1a702cd4..e706d9ef881 100644--- a/Gemfile+++ b/Gemfile@@ -236,6 +236,7 @@ gem "sshkey", require: falsegem "lz4-ruby", require: false, platform: :rubygem "sanitize"+gem "reverse_markdown", "3.0.2", require: falseif ENV["IMPORT"] == "1"gem "mysql2"@@ -244,7 +245,6 @@ if ENV["IMPORT"] == "1"# NOTE: in import mode the version of sqlite can matter a lot, so we stick it to a specific onegem "sqlite3", "~> 1.3", ">= 1.3.13"gem "ruby-bbcode-to-md", git: "https://github.com/nlalonde/ruby-bbcode-to-md"- gem "reverse_markdown"gem "tiny_tds"gem "csv"enddiff --git a/Gemfile.lock b/Gemfile.lockindex b768dfc1e68..01687a44517 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -583,6 +583,8 @@ GEMio-console (~> 0.5)request_store (1.7.0)rack (>= 1.4)+ reverse_markdown (3.0.2)+ nokogirirexml (3.4.4)rinku (2.0.6)rotp (6.3.0)@@ -948,6 +950,7 @@ DEPENDENCIESredcarpetredisredis-namespace+ reverse_markdown (= 3.0.2)rinkurotprqrcode@@ -1239,6 +1242,7 @@ CHECKSUMSregexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebbreline (0.7.0) sha256=5b012d8e55dbf9d450f12bde2cf7d15ff546ae80b3f8f3b30e570d431815583drequest_store (1.7.0) sha256=e1b75d5346a315f452242a68c937ef8e48b215b9453a77a6c0acdca2934c88cb+ reverse_markdown (3.0.2) sha256=818ebb92ce39dbb1a291690dd1ec9a6d62530d4725296b17e9c8f668f9a5b8afrexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142rinku (2.0.6) sha256=8b60670e3143f3db2b37efa262971ce3619ec23092045498ef9f077d82828d7drotp (6.3.0) sha256=75d40087e65ed0d8022c33055a6306c1c400d1c12261932533b5d6cbcd868854diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rbindex 353f53db305..0bfc94cd6d2 100644--- a/app/controllers/categories_controller.rb+++ b/app/controllers/categories_controller.rb@@ -71,6 +71,14 @@ class CategoriesController < ApplicationControllerendformat.json { render_serialized(@category_list, CategoryListSerializer) }+ format.md do+ render_markdown(+ MarkdownEndpoint::DirectoryRenderer.new.categories(+ @category_list,+ params.permit(:page, :parent_category_id, :include_subcategories, :tag).to_h,+ ),+ )+ endendend@@ -870,6 +878,7 @@ class CategoriesController < ApplicationControllerparent_category_id: parent_category&.id,include_topics: include_topics,include_subcategories: include_subcategories,+ include_pagination: request.format.md?,tag: params[:tag],page: params[:page].try(:to_i) || 1,}diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rbindex 650356e624e..a9130246b86 100644--- a/app/controllers/tags_controller.rb+++ b/app/controllers/tags_controller.rb@@ -98,6 +98,7 @@ class TagsController < ::ApplicationControllerformat.html { render :index }format.json { render json: { tags: @tags, extras: @extras } }+ format.md { render_markdown(MarkdownEndpoint::DirectoryRenderer.new.tags(@tags, @extras)) }endend@@ -556,7 +557,14 @@ class TagsController < ::ApplicationControllerendurl += ".json" if request.format.json?- url += "?#{request.query_string}" if request.query_string.present?+ if request.format.md?+ url += ".md"+ query =+ request.query_parameters.slice(*MarkdownEndpoint::ControllerSupport::SAFE_QUERY_PARAMETERS)+ url += "?#{query.to_query}" if query.present?+ elsif request.query_string.present?+ url += "?#{request.query_string}"+ endredirect_to url, status: :moved_permanentlyenddiff --git a/app/controllers/topics_controller.rb b/app/controllers/topics_controller.rbindex 94a0fbc468d..252eab8e777 100644--- a/app/controllers/topics_controller.rb+++ b/app/controllers/topics_controller.rb@@ -194,6 +194,7 @@ class TopicsController < ApplicationControllerlast_page = visible_posts_count > 0 ? ((visible_posts_count - 1) / chunk_size) + 1 : 1url = @topic_view.topic.relative_urlurl += ".json" if request.format.json?+ url += ".md" if request.format.md?url += "?page=#{last_page}" if last_page > 1return redirect_to url, status: :moved_permanentlyend@@ -1478,6 +1479,7 @@ class TopicsController < ApplicationControllerurl = topic.relative_urlurl << "/#{post_number}" if post_number.to_i > 0url << ".json" if request.format.json?+ url << ".md" if request.format.md?opts.each do |k, v|s = url.include?("?") ? "&" : "?"@@ -1543,6 +1545,8 @@ class TopicsController < ApplicationControllerdef perform_show_responseif request.head?+ response.content_type = "text/markdown; charset=utf-8" if request.format.md?+ merge_vary_accept if request.format.md?head :okreturnend@@ -1575,6 +1579,19 @@ class TopicsController < ApplicationControllerendformat.json { render_json_dump(topic_view_serializer) }+ format.md do+ render_markdown(+ MarkdownEndpoint::TopicRenderer.new(+ @topic_view,+ guardian: guardian,+ post_number: params[:post_number],+ query:+ request.query_parameters.slice(+ *MarkdownEndpoint::ControllerSupport::SAFE_QUERY_PARAMETERS,+ ),+ ).render,+ )+ endendenddiff --git a/app/models/category_list.rb b/app/models/category_list.rbindex be1d82664ac..97fd377320b 100644--- a/app/models/category_list.rb+++ b/app/models/category_list.rb@@ -13,6 +13,7 @@ class CategoryListself.preloaded_topic_custom_fields = Set.newattr_accessor :categories, :uncategorized+ attr_reader :next_pagedef self.register_included_association(association)@included_associations ||= []@@ -167,6 +168,10 @@ class CategoryList@categories = query.to_a+ if paginate && @options[:include_pagination] && query.offset(page * CATEGORIES_PER_PAGE).exists?+ @next_page = page + 1+ end+if paginate && @options[:parent_category_id].blank?categories_with_rownum =Categorydiff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erbindex 5fcb3573082..3a5abba1426 100644--- a/app/views/layouts/application.html.erb+++ b/app/views/layouts/application.html.erb@@ -88,6 +88,7 @@<%- end %><%= yield :head %>+ <%= markdown_alternate_link_tag %><%= build_plugin_html 'server:before-head-close' %>diff --git a/app/views/layouts/crawler.html.erb b/app/views/layouts/crawler.html.erbindex 5f1de923f99..e422538e025 100644--- a/app/views/layouts/crawler.html.erb+++ b/app/views/layouts/crawler.html.erb@@ -10,6 +10,7 @@<%= theme_lookup("head_tag") %><%= render_google_universal_analytics_code %><%= yield :head %>+ <%= markdown_alternate_link_tag %><%= build_plugin_html 'server:before-head-close-crawler' %></head><body class="crawler <% if show_browser_update? %>browser-update<% end %>">diff --git a/app/views/layouts/no_ember.html.erb b/app/views/layouts/no_ember.html.erbindex b175662f909..02630f820d7 100644--- a/app/views/layouts/no_ember.html.erb+++ b/app/views/layouts/no_ember.html.erb@@ -13,6 +13,7 @@<%- end %><%= yield(:no_ember_head) %>+ <%= markdown_alternate_link_tag %><%- if allow_plugins? %><%= build_plugin_html 'server:before-head-close' %>diff --git a/app/views/list/list.rss.erb b/app/views/list/list.rss.erbindex 9e3dfc88294..c29c90afd4a 100644--- a/app/views/list/list.rss.erb+++ b/app/views/list/list.rss.erb@@ -7,9 +7,10 @@<link><%= @link %></link><description><%= @description %></description><% if lang %><language><%= lang.sub('_', '-')%></language><% end %>+ <atom:link href="<%= @atom_link %>" rel="self" type="application/rss+xml" />+ <% if markdown_alternate_url %><atom:link href="<%= markdown_alternate_url %>" rel="alternate" type="text/markdown" /><% end %><% if @topic_list.topics && @topic_list.topics.length > 0 %><lastBuildDate><%= @topic_list.topics.first.created_at.rfc2822 %></lastBuildDate>- <atom:link href="<%= @atom_link %>" rel="self" type="application/rss+xml" /><% @topic_list.topics.each do |topic| %><% topic_url = topic.url -%><item>diff --git a/app/views/topics/show.rss.erb b/app/views/topics/show.rss.erbindex bbb4989b4f6..1170d8dffc3 100644--- a/app/views/topics/show.rss.erb+++ b/app/views/topics/show.rss.erb@@ -11,6 +11,7 @@<lastBuildDate><%= @topic_view.topic.bumped_at.rfc2822 %></lastBuildDate><category><%= @topic_view.topic.category.name %></category><atom:link href="<%= topic_url %>.rss" rel="self" type="application/rss+xml" />+ <% if markdown_alternate_url %><atom:link href="<%= markdown_alternate_url %>" rel="alternate" type="text/markdown" /><% end %><% @topic_view.recent_posts.each do |post| %><% next unless post.user %><item>diff --git a/config/initializers/markdown_endpoints.rb b/config/initializers/markdown_endpoints.rbnew file mode 100644index 00000000000..e0428bc2633--- /dev/null+++ b/config/initializers/markdown_endpoints.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++require "reverse_markdown"+require "markdown_endpoint/accept_header"+require "markdown_endpoint/request_constraint"+require "markdown_endpoint/vary_middleware"+require "markdown_endpoint/cooked_processor"+require "markdown_endpoint/topic_renderer"+require "markdown_endpoint/topic_list_renderer"+require "markdown_endpoint/directory_renderer"+require "markdown_endpoint/controller_support"++Mime::Type.register("text/markdown", :md) unless Mime::Type.lookup_by_extension(:md)+Rails.application.config.middleware.insert_before(0, MarkdownEndpoint::VaryMiddleware)++Rails.application.config.to_prepare do+ ApplicationController.include(MarkdownEndpoint::ControllerSupport)+enddiff --git a/config/locales/server.en.yml b/config/locales/server.en.ymlindex dbf5a4cb1ad..c512a61de58 100644--- a/config/locales/server.en.yml+++ b/config/locales/server.en.yml@@ -1,4 +1,16 @@en:+ markdown_endpoints:+ latest: "Latest"+ categories: "Categories"+ tags: "Tags"+ parent: "Parent: %{parent}"+ previous_page: "Previous page"+ next_page: "Next page"+ empty: "No entries to display."+ replies: "Replies"+ post_author: "Author: %{author}"+ post_date: "Post date: %{timestamp}"+ last_updated: "Last updated"mcp:errors:draft_sequence_conflict: "Draft sequence conflict"@@ -3165,6 +3177,7 @@ en:enable_invite_modal_with_roles: "Enables a redesigned invite flow that lets admins invite someone directly as an admin. The invitee becomes a moderator on signup and an admin once the inviter confirms via email."update_pending_users_reminder_default: "We are updating the default for the {{setting:pending_users_reminder_delay_minutes}} setting to 30 minutes so that staff are promptly notified when a new member is waiting for approval. If you have previously changed this setting, this change will have no effect."enable_generated_llms_txt: "Generates a concise /llms.txt when no custom file is uploaded. The document describes public discovery routes, the preferred Discourse MCP interface, and responsible agent access. To tune automated access, configure {{setting:slow_down_crawler_user_agents}} and {{setting:slow_down_crawler_rate}}."+ experimental_markdown_endpoints: "Serve supported topics and topic lists as Markdown through .md URLs and Accept negotiation. Normal visibility and profile privacy rules always apply."errors:invalid_css_color: "Invalid color. Enter a color name or hex value."diff --git a/config/routes.rb b/config/routes.rbindex adaea9575fc..6bb0cd90e8d 100644--- a/config/routes.rb+++ b/config/routes.rb@@ -2,6 +2,8 @@require "sidekiq/web"require "mini_scheduler/web"+require_relative "../lib/markdown_endpoint/accept_header"+require_relative "../lib/markdown_endpoint/request_constraint"# The following constants have been replaced with `RouteFormat` and are deprecated.USERNAME_ROUTE_FORMAT = /[%\w.\-]+?/ unless defined?(USERNAME_ROUTE_FORMAT)@@ -11,6 +13,50 @@ Discourse::Application.routes.draw dodef patch(*)end # Disable PATCH requests+ constraints MarkdownEndpoint::RequestConstraint.new do+ get "/",+ to: "list#latest",+ format: false,+ defaults: {+ format: :md,+ },+ constraints: MarkdownEndpoint::RequestConstraint.new(accept: true)++ scope format: false,+ constraints: {+ topic_id: /\d+/,+ post_number: /\d+/,+ tag_id: /\d+/,+ category_slug_path_with_id: %r{.+/\d+},+ } do+ {+ "latest" => "list#latest",+ "new" => "list#new",+ "unread" => "list#unread",+ "hot" => "list#hot",+ "top" => "list#top",+ "categories" => "categories#index",+ "tags" => "tags#index",+ "c/*category_slug_path_with_id" => "list#category_default",+ "tag/:tag_slug/:tag_id" => "tags#show",+ "tag/:tag_name" => "tags#show",+ "t/:topic_id/:post_number" => "topics#show",+ "t/:topic_id" => "topics#show",+ "t/:slug/:topic_id/:post_number" => "topics#show",+ "t/:slug/:topic_id" => "topics#show",+ }.each do |path, action|+ # Require the format so HTML URL generation cannot select a Markdown route.+ get "#{path}.:format", to: action, constraints: { format: /md/ }+ get path,+ to: action,+ defaults: {+ format: :md,+ },+ constraints: MarkdownEndpoint::RequestConstraint.new(accept: true)+ end+ end+ end+scope path: nil, constraints: { format: %r{(json|html|\*/\*)} } dorelative_url_root =(@@ -1893,33 +1939,38 @@ Discourse::Application.routes.draw doresources :tag_groups, constraints: StaffConstraint.new, except: [:edit]get "/tag_groups/filter/search" => "tag_groups#search", :format => :json- Discourse.filters.each do |filter|- root to: "list##{filter}",- constraints: HomePageConstraint.new("#{filter}"),- as: "list_#{filter}"- end+ # Allow the controller to fall back to HTML when Markdown loses Accept negotiation.+ scope constraints: { format: %r{(json|html|markdown|\*/\*)} } do+ Discourse.filters.each do |filter|+ root to: "list##{filter}",+ constraints: HomePageConstraint.new("#{filter}"),+ as: "list_#{filter}"+ end- DiscoursePluginRegistry._raw_homepage_options.each do |registration|- option = registration[:value]- get "/", to: option[:route], constraints: HomePageConstraint.new(option[:id])- end+ DiscoursePluginRegistry._raw_homepage_options.each do |registration|+ option = registration[:value]+ get "/", to: option[:route], constraints: HomePageConstraint.new(option[:id])+ end- get "/t/:topic_id/view-stats.json" => "topic_view_stats#index"+ # special case for categories+ root to: "categories#index",+ constraints: HomePageConstraint.new("categories"),+ as: "categories_index"- # special case for categories- root to: "categories#index",- constraints: HomePageConstraint.new("categories"),- as: "categories_index"+ root to: "finish_installation#index",+ constraints: HomePageConstraint.new("finish_installation"),+ as: "installation_redirect"- root to: "finish_installation#index",- constraints: HomePageConstraint.new("finish_installation"),- as: "installation_redirect"+ root to: "home_page#custom",+ constraints: HomePageConstraint.new("custom"),+ as: "home_page_custom"- root to: "home_page#custom",- constraints: HomePageConstraint.new("custom"),- as: "home_page_custom"+ root to: "home_page#blank",+ constraints: HomePageConstraint.new("blank"),+ as: "home_page_blank"+ end- root to: "home_page#blank", constraints: HomePageConstraint.new("blank"), as: "home_page_blank"+ get "/t/:topic_id/view-stats.json" => "topic_view_stats#index"get "/custom" => "home_page#custom"diff --git a/config/site_settings.yml b/config/site_settings.ymlindex b7392f21b42..3974b7eb079 100644--- a/config/site_settings.yml+++ b/config/site_settings.yml@@ -4781,6 +4781,10 @@ dashboard:client: trueexperimental:+ experimental_markdown_endpoints:+ default: false+ hidden: true+ area: "experimental"enable_generated_llms_txt:default: falsehidden: truediff --git a/lib/markdown_endpoint/accept_header.rb b/lib/markdown_endpoint/accept_header.rbnew file mode 100644index 00000000000..08df532e9fc--- /dev/null+++ b/lib/markdown_endpoint/accept_header.rb@@ -0,0 +1,43 @@+# frozen_string_literal: true++module MarkdownEndpoint+ module AcceptHeader+ module_function++ def quality(header, type, subtype)+ return 0.0 if header.blank?++ header.split(",").filter_map { |entry| match(entry, type, subtype) }.max&.last || 0.0+ end++ def preferred?(header)+ markdown = quality(header, "text", "markdown")+ alternatives = [quality(header, "text", "html"), quality(header, "application", "json")]+ markdown.positive? && markdown > alternatives.max+ end++ def match(entry, type, subtype)+ media_type, *parameters = entry.strip.split(";")+ candidate_type, candidate_subtype = media_type.to_s.downcase.strip.split("/", 2)+ return if candidate_subtype.blank?++ specificity =+ if candidate_type == type && candidate_subtype == subtype+ 2+ elsif candidate_type == type && candidate_subtype == "*"+ 1+ elsif candidate_type == "*" && candidate_subtype == "*"+ 0+ end+ return unless specificity++ quality = 1.0+ parameters.each do |parameter|+ name, value = parameter.strip.split("=", 2)+ quality = value.to_f.clamp(0.0, 1.0) if name&.casecmp?("q")+ end+ [specificity, quality]+ end+ private_class_method :match+ end+enddiff --git a/lib/markdown_endpoint/controller_support.rb b/lib/markdown_endpoint/controller_support.rbnew file mode 100644index 00000000000..fcf15174a05--- /dev/null+++ b/lib/markdown_endpoint/controller_support.rb@@ -0,0 +1,142 @@+# frozen_string_literal: true++module MarkdownEndpoint+ module ControllerSupport+ extend ActiveSupport::Concern++ SAFE_QUERY_PARAMETERS = %w[+ ascending+ before+ bumped_before+ category+ exclude_tag+ f+ filter+ filter_top_level_replies+ filter_upwards_post_id+ group_name+ include_subcategories+ match_all_tags+ max_posts+ min_posts+ no_subcategories+ no_tags+ order+ page+ parent_category_id+ per_page+ period+ post_number+ q+ replies_to_post_number+ search+ show_deleted+ state+ status+ subset+ tag+ tags+ topic_ids+ username_filters+ ].freeze++ included do+ before_action :normalize_markdown_format+ after_action :add_markdown_discovery_header+ helper_method :markdown_alternate_link_tag, :markdown_alternate_url+ end++ def markdown_alternate_url+ return unless SiteSetting.experimental_markdown_endpoints++ path = markdown_alternate_path+ return if path.blank?++ query = request.query_parameters.slice(*SAFE_QUERY_PARAMETERS)+ url = "#{Discourse.base_url_no_prefix}#{path}"+ query.present? ? "#{url}?#{query.to_query}" : url+ end++ def markdown_alternate_link_tag+ return if request.format.md?++ url = markdown_alternate_url+ return if url.blank?++ helpers.tag.link(rel: "alternate", type: "text/markdown", href: url)+ end++ def render_markdown(body)+ merge_vary_accept+ render plain: body, content_type: "text/markdown; charset=utf-8"+ end++ private++ def normalize_markdown_format+ return unless request.format.md?+ return if request.path.end_with?(".md")++ accept = request.headers["Accept"]+ if SiteSetting.experimental_markdown_endpoints && markdown_supported_response? &&+ AcceptHeader.preferred?(accept)+ return+ end++ html_quality = AcceptHeader.quality(accept, "text", "html")+ json_quality = AcceptHeader.quality(accept, "application", "json")+ request.formats = [json_quality > html_quality ? :json : :html]+ end++ def add_markdown_discovery_header+ return unless response.status.between?(200, 299)+ return unless response.media_type == "text/html"+ return if markdown_alternate_url.blank?++ value = %(<#{markdown_alternate_url}>; rel="alternate"; type="text/markdown")+ response.headers["Link"] = [response.headers["Link"], value].compact.join(", ")+ merge_vary_accept+ end++ def merge_vary_accept+ tokens = response.headers["Vary"].to_s.split(",").map(&:strip).reject(&:blank?)+ tokens << "Accept" unless tokens.any? { |token| token.casecmp?("Accept") }+ response.headers["Vary"] = tokens.join(", ")+ end++ def markdown_supported_response?+ markdown_alternate_path.present?+ end++ def markdown_alternate_path+ app_path = request.path.delete_prefix(Discourse.base_path)+ return "#{Discourse.base_path}/latest.md" if app_path.blank? || app_path == "/"+ return if app_path.end_with?(".json")+ route_path = app_path.delete_suffix(".md")++ supported =+ case controller_name+ when "topics"+ %w[show feed].include?(action_name) &&+ route_path.match?(%r{\A/t/(?:[^/]+/)?\d+(?:/\d+)?(?:\.rss)?\z})+ when "list"+ route_path.match?(%r{\A/(?:latest|new|unread|hot|top)(?:\.rss)?\z}) ||+ route_path.match?(%r{\A/c/.+/\d+(?:\.rss)?\z})+ when "tags"+ (action_name == "index" && route_path == "/tags") ||+ (+ %w[show tag_feed].include?(action_name) &&+ route_path.match?(%r{\A/tag/[^/]+(?:/\d+)?(?:\.rss)?\z})+ )+ when "categories"+ action_name == "index" && route_path == "/categories"+ else+ false+ end+ return unless supported++ "#{Discourse.base_path}#{route_path.delete_suffix(".rss").chomp("/")}.md"+ end+ end+enddiff --git a/lib/markdown_endpoint/cooked_processor.rb b/lib/markdown_endpoint/cooked_processor.rbnew file mode 100644index 00000000000..82a8ec2ba01--- /dev/null+++ b/lib/markdown_endpoint/cooked_processor.rb@@ -0,0 +1,222 @@+# frozen_string_literal: true++module MarkdownEndpoint+ class CookedProcessor+ BLOCK_TAG = "discourse-markdown-block"+ INLINE_TAG = "discourse-markdown-inline"+ VERSION = 2++ class PreservedBlockConverter < ReverseMarkdown::Converters::Base+ def convert(node, _state = {})+ "\n\n#{node.text}\n\n"+ end+ end++ class PreservedInlineConverter < ReverseMarkdown::Converters::Base+ def convert(node, _state = {})+ node.text+ end+ end++ ReverseMarkdown::Converters.register(BLOCK_TAG.to_sym, PreservedBlockConverter.new)+ ReverseMarkdown::Converters.register(INLINE_TAG.to_sym, PreservedInlineConverter.new)++ def self.to_markdown(cooked_html)+ new(cooked_html).to_markdown+ end++ def initialize(cooked_html)+ @fragment = Nokogiri::HTML5.fragment(cooked_html.to_s)+ end++ def to_markdown+ strip_chrome+ replace_emojis+ replace_mentions+ replace_hashtags+ replace_lightboxes+ replace_code_blocks+ replace_quotes+ replace_oneboxes+ replace_details+ replace_polls+ absolutize_urls++ ReverseMarkdown.convert(@fragment.to_html, github_flavored: true, unknown_tags: :bypass).strip+ end++ private++ def strip_chrome+ @fragment.css("div.meta, div.quote-controls").each(&:remove)+ end++ def replace_emojis+ @fragment+ .css("img.emoji")+ .each { |image| image.replace(preserved_inline(image["title"] || image["alt"] || "")) }+ end++ def replace_mentions+ @fragment+ .css("a.mention")+ .each do |anchor|+ anchor.replace(preserved_inline("@#{anchor.text.to_s.delete_prefix("@").strip}"))+ end+ end++ def replace_hashtags+ @fragment+ .css("a.hashtag, a.hashtag-cooked")+ .each do |anchor|+ anchor.replace(preserved_inline("##{anchor.text.to_s.delete_prefix("#").strip}"))+ end+ end++ def replace_lightboxes+ @fragment+ .css("a.lightbox")+ .each do |anchor|+ image = anchor.at_css("img")+ if image+ image["src"] = anchor["href"] if anchor["href"].present?+ anchor.replace(image)+ else+ anchor.remove+ end+ end++ @fragment+ .css("div.lightbox-wrapper")+ .each do |wrapper|+ content = wrapper.at_css("img, picture")+ content ? wrapper.replace(content) : wrapper.remove+ end+ end++ def replace_code_blocks+ @fragment+ .css("pre > code")+ .each do |code|+ language = code["class"].to_s.split.find { |name| name.match?(/\A(?:lang|language)-/) }+ language = language&.sub(/\A(?:lang|language)-/, "")+ content = code.text+ longest_run = content.scan(/`+/).map(&:length).max.to_i+ fence = "`" * [3, longest_run + 1].max+ opening = language.present? ? "#{fence}#{language}" : fence+ code.parent.replace(preserved_block("#{opening}\n#{content}\n#{fence}"))+ end+ end++ def replace_quotes+ @fragment+ .css("aside.quote")+ .each do |quote|+ attribution = quote.at_css(".title a")+ username =+ attribution&.text&.strip&.delete_suffix(":").presence || quote["data-username"].to_s+ url = absolute_url(attribution&.[]("href") || "#")+ body = convert_html(quote.at_css("blockquote")&.inner_html)+ lines = ["> [@#{escape_text(username)}](#{url}):", ">"]+ body.each_line { |line| lines << "> #{line.chomp}" }+ quote.replace(preserved_block(lines.join("\n")))+ end+ end++ def replace_oneboxes+ @fragment+ .css("aside.onebox")+ .each do |onebox|+ title_link = onebox.at_css("h3 a")+ title = title_link&.text&.strip+ url =+ title_link&.[]("href") || onebox.at_css("header.source a")&.[]("href") ||+ onebox.at_css("article a")&.[]("href")+ excerpt_node = onebox.at_css("article.onebox-body p, .onebox-body p")+ excerpt =+ if excerpt_node+ excerpt_node = excerpt_node.dup+ excerpt_node.css("br").each { |line_break| line_break.replace("\n") }+ excerpt_node.text.strip+ end+ lines = []+ if url.present? && title.present?+ lines << "> **[#{escape_text(title)}](#{absolute_url(url)})**"+ elsif url.present?+ lines << "> <#{absolute_url(url)}>"+ end+ if excerpt.present?+ lines << ">"+ excerpt.each_line { |line| lines << "> #{escape_text(line.chomp)}" }+ end+ lines.empty? ? onebox.remove : onebox.replace(preserved_block(lines.join("\n")))+ end+ end++ def replace_details+ @fragment+ .css("details")+ .each do |details|+ summary = escape_text(details.at_css("summary")&.text&.strip.to_s)+ body_html = details.children.reject { |node| node.name == "summary" }.map(&:to_html).join+ body = convert_html(body_html)+ lines = ["> **#{summary}**", ">"]+ body.each_line { |line| lines << "> #{line.chomp}" }+ details.replace(preserved_block(lines.join("\n")))+ end+ end++ def replace_polls+ @fragment+ .css("div.poll")+ .each do |poll|+ title =+ poll["data-poll-title"].presence ||+ poll.at_css("[data-poll-title]")&.[]("data-poll-title").presence || "Poll"+ poll.replace(preserved_block("_Poll: #{escape_text(title)} (view on site)_"))+ end+ end++ def absolutize_urls+ @fragment+ .css("[href], [src]")+ .each do |node|+ node["href"] = absolute_url(node["href"]) if node["href"].present?+ node["src"] = absolute_url(node["src"], cdn: true) if node["src"].present?+ end+ end++ def absolute_url(url, cdn: false)+ return url if url.blank? || url.start_with?("#", "mailto:", "data:")+ return "#{Discourse.base_url.split(":", 2).first}:#{url}" if url.start_with?("//")+ return url if url.match?(/\A[a-z][a-z0-9+.-]*:/i)+ return UrlHelper.absolute(url, cdn ? Discourse.asset_host : nil) if url.start_with?("/")++ "#{Discourse.base_url}/#{url}"+ end++ def convert_html(html)+ fragment = self.class.new(html)+ fragment.to_markdown+ end++ def escape_text(text)+ text+ .to_s+ .gsub(/[\\`*_\[\]<>]/) { |character| "\\#{character}" }+ .sub(/\A[#>-]/) { |character| "\\#{character}" }+ end++ def preserved_inline(content)+ node = Nokogiri::XML::Node.new(INLINE_TAG, @fragment.document)+ node.content = content+ node+ end++ def preserved_block(content)+ node = Nokogiri::XML::Node.new(BLOCK_TAG, @fragment.document)+ node.content = content+ node+ end+ end+enddiff --git a/lib/markdown_endpoint/directory_renderer.rb b/lib/markdown_endpoint/directory_renderer.rbnew file mode 100644index 00000000000..d651865f9f9--- /dev/null+++ b/lib/markdown_endpoint/directory_renderer.rb@@ -0,0 +1,84 @@+# frozen_string_literal: true++module MarkdownEndpoint+ class DirectoryRenderer+ def self.navigation+ links = [+ "[#{I18n.t("markdown_endpoints.latest")}](#{Discourse.base_url}/latest.md)",+ "[#{I18n.t("markdown_endpoints.categories")}](#{Discourse.base_url}/categories.md)",+ ]+ if SiteSetting.tagging_enabled+ links << "[#{I18n.t("markdown_endpoints.tags")}](#{Discourse.base_url}/tags.md)"+ end+ links.join(" · ")+ end++ def categories(category_list, params = {})+ records =+ category_list.categories.flat_map { |category| category_entries(category) }.uniq(&:id)+ parents = records.index_by(&:id)+ lines = header("categories", records)+ records.each do |category|+ append_entry(lines, category.name, category.url, category.description)+ if parent = parents[category.parent_category_id]+ lines.concat(+ ["", I18n.t("markdown_endpoints.parent", parent: link(parent.name, parent.url))],+ )+ end+ end++ page = [params["page"].to_i, 1].max+ lines.concat(["", page_link("previous_page", params, page - 1)]) if page > 1+ if category_list.next_page+ lines.concat(["", page_link("next_page", params, category_list.next_page)])+ end+ "#{lines.join("\n")}\n"+ end++ def tags(tags, extras)+ records =+ (tags + extras.values.flatten.flat_map { |group| group[:tags] }).uniq { |tag| tag[:id] }+ lines = header("tags", records)+ records.each do |tag|+ append_entry(+ lines,+ tag[:name],+ "#{Discourse.base_path}/tag/#{tag[:slug]}/#{tag[:id]}",+ tag[:description],+ )+ end+ "#{lines.join("\n")}\n"+ end++ private++ def category_entries(category)+ [category, *Array(category.subcategory_list).flat_map { |child| category_entries(child) }]+ end++ def header(kind, records)+ lines = ["# #{I18n.t("markdown_endpoints.#{kind}")}", "", self.class.navigation]+ lines.concat(["", I18n.t("markdown_endpoints.empty")]) if records.empty?+ lines+ end++ def append_entry(lines, name, url, description)+ lines.concat(["", "## #{link(name, url)}"])+ excerpt = Nokogiri::HTML5.fragment(description.to_s).text.squish.truncate(200)+ lines.concat(["", escape_text(excerpt)]) if excerpt.present?+ end++ def link(name, url)+ "[#{escape_text(name)}](#{Discourse.base_url_no_prefix}#{url}.md)"+ end++ def page_link(label, params, page)+ query = params.merge("page" => page).to_query+ "[#{I18n.t("markdown_endpoints.#{label}")}](#{Discourse.base_url}/categories.md?#{query})"+ end++ def escape_text(text)+ text.to_s.gsub(/[\\`*_\[\]<>]/) { |character| "\\#{character}" }+ end+ end+enddiff --git a/lib/markdown_endpoint/request_constraint.rb b/lib/markdown_endpoint/request_constraint.rbnew file mode 100644index 00000000000..749d9556908--- /dev/null+++ b/lib/markdown_endpoint/request_constraint.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++module MarkdownEndpoint+ class RequestConstraint+ VARY_ACCEPT_ENV_KEY = "discourse.markdown_endpoint.vary_accept"++ def initialize(accept: false)+ @accept = accept+ end++ def matches?(request)+ return false unless SiteSetting.experimental_markdown_endpoints+ return true unless @accept+ return false if request.path.end_with?(".json", ".rss")++ request.env[VARY_ACCEPT_ENV_KEY] = true+ AcceptHeader.preferred?(request.headers["Accept"])+ end+ end+enddiff --git a/lib/markdown_endpoint/timestamp.rb b/lib/markdown_endpoint/timestamp.rbnew file mode 100644index 00000000000..caf64f838a6--- /dev/null+++ b/lib/markdown_endpoint/timestamp.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++module MarkdownEndpoint+ class Timestamp+ def initialize(user)+ @timezone = Time.find_zone(user&.user_option&.timezone) || Time.zone+ end++ def render(time, url:)+ local = time.in_time_zone(@timezone)+ label =+ "#{I18n.l(local, format: :long)} #{local.strftime("%Z")}"+ .squish+ .gsub(/[\\`*_\[\]<>]/) { |character| "\\#{character}" }+ %([#{label}](#{url} "#{local.iso8601}"))+ end+ end+enddiff --git a/lib/markdown_endpoint/topic_list_renderer.rb b/lib/markdown_endpoint/topic_list_renderer.rbnew file mode 100644index 00000000000..7eb05033f94--- /dev/null+++ b/lib/markdown_endpoint/topic_list_renderer.rb@@ -0,0 +1,87 @@+# frozen_string_literal: true++module MarkdownEndpoint+ class TopicListRenderer+ def initialize(+ topics:,+ title:,+ url:,+ page:,+ next_page_url: nil,+ previous_page_url: nil,+ user: nil+ )+ @topics = topics.to_a+ @title = title+ @url = url+ @page = page.to_i+ @next_page_url = next_page_url+ @previous_page_url = previous_page_url+ @timestamp = Timestamp.new(user)+ preload_users+ end++ def render+ buffer = String.new(header)+ @topics.each do |topic|+ buffer << "\n\n---\n\n"+ buffer << render_topic(topic)+ end+ {+ "previous_page" => @previous_page_url,+ "next_page" => @next_page_url,+ }.each { |label, target| buffer << "\n\n#{page_link(label, target)}" if target.present? }+ buffer << "\n"+ end++ private++ def page_link(label, target)+ url = URI(@url)+ query = Rack::Utils.parse_nested_query(url.query).except("page")+ query.merge!(+ Rack::Utils.parse_nested_query(URI(target).query).slice(+ *ControllerSupport::SAFE_QUERY_PARAMETERS,+ ),+ )+ url.query = query.to_query.presence+ "[#{I18n.t("markdown_endpoints.#{label}")}](#{url})"+ end++ def preload_users+ ActiveRecord::Associations::Preloader.new(records: @topics, associations: :user).call+ end++ def header+ lines = ["# #{escape_text(@title)}", "", "**URL:** #{@url}"]+ lines.concat(["", DirectoryRenderer.navigation])+ lines.concat(["", "**Page:** #{@page + 1}"]) if @page.positive?+ lines.join("\n")+ end++ def render_topic(topic)+ lines = ["## [#{escape_text(topic.title)}](#{topic.url})"]+ lines.concat(["", '<div class="topic-metadata">', ""])+ metadata = []+ if topic.user+ metadata << "**Author:** [@#{escape_text(topic.user.username)}](#{topic.user.full_url})"+ end+ metadata << "**#{I18n.t("markdown_endpoints.replies")}:** #{[topic.posts_count - 1, 0].max}"+ if topic.last_posted_at+ metadata << "**#{I18n.t("markdown_endpoints.last_updated")}:** #{@timestamp.render(topic.last_posted_at, url: topic.url)}"+ end+ lines << metadata.join("\\\n")+ lines.concat(["", "</div>"])+ excerpt = Nokogiri::HTML5.fragment(topic.excerpt.to_s).text.squish+ lines.concat(["", escape_text(excerpt)]) if excerpt.present?+ lines.join("\n")+ end++ def escape_text(text)+ text+ .to_s+ .gsub(/[\\`*_\[\]<>]/) { |character| "\\#{character}" }+ .sub(/\A[#>-]/) { |character| "\\#{character}" }+ end+ end+enddiff --git a/lib/markdown_endpoint/topic_renderer.rb b/lib/markdown_endpoint/topic_renderer.rbnew file mode 100644index 00000000000..217483a3902--- /dev/null+++ b/lib/markdown_endpoint/topic_renderer.rb@@ -0,0 +1,115 @@+# frozen_string_literal: true++require "digest"++module MarkdownEndpoint+ class TopicRenderer+ def initialize(topic_view, guardian:, post_number: nil, query: {})+ @topic_view = topic_view+ @topic = topic_view.topic+ @guardian = guardian+ @post_number = post_number&.to_i+ @query = query+ @timestamp = Timestamp.new(guardian.user)+ end++ def render+ posts = visible_posts+ raise Discourse::NotFound if single_post? && posts.empty?++ buffer = String.new(header(posts.length))+ posts.each_with_index do |post, index|+ buffer << (index.zero? ? "\n\n" : "\n\n---\n\n")+ buffer << render_post(post)+ end+ buffer << "\n\n---\n\n_[View the full topic](#{@topic.url})._" if single_post?+ unless single_post?+ {+ "previous_page" => @topic_view.prev_page,+ "next_page" => @topic_view.next_page,+ }.each do |label, page|+ next unless page++ query = @query.merge("page" => page).to_query+ buffer << "\n\n[#{I18n.t("markdown_endpoints.#{label}")}](#{@topic.url}.md?#{query})"+ end+ end+ buffer << "\n"+ end++ private++ def visible_posts+ posts = @topic_view.posts.to_a+ posts = posts.select { |post| post.post_number == @post_number } if single_post?+ posts =+ posts.reject { |post| post.post_type == Post.types[:whisper] } unless can_see_whispers?+ posts+ end++ def single_post?+ @post_number.present?+ end++ def can_see_whispers?+ user = @guardian.user+ allowed_group_ids = SiteSetting.whispers_allowed_groups_map+ user && allowed_group_ids.present? && user.group_users.exists?(group_id: allowed_group_ids)+ end++ def header(visible_post_count)+ title = ContentLocalization.translated_topic_title(@topic, @guardian) || @topic.title+ lines = ["# #{escape_text(title)}", "", "**URL:** #{@topic.url}"]+ lines << "**Category:** #{escape_text(@topic.category.name)}" if @topic.category+ tags = @topic_view.visible_tags.map(&:name)+ lines << "**Tags:** #{tags.map { |tag| escape_text(tag) }.join(", ")}" if tags.present?+ lines << "**Created:** #{@timestamp.render(@topic.created_at, url: @topic.url)}"+ lines << "**Posts on this page:** #{visible_post_count}"+ lines << "**Page:** #{@topic_view.page}" unless single_post?+ lines << "**Showing post:** #{@post_number}" if single_post?+ lines.join("\n")+ end++ def render_post(post)+ serializer = BasicPostSerializer.new(post, scope: @guardian, root: false)+ serializer.topic_view = @topic_view+ cooked = serializer.cooked.to_s+ lines = ['<div class="post-metadata">', ""]+ if post.user+ avatar_url = UrlHelper.absolute(post.user.avatar_template.gsub("{size}", "32"))+ avatar_url = "#{Discourse.base_protocol}:#{avatar_url}" if avatar_url.start_with?("//")+ avatar_url = URI::DEFAULT_PARSER.escape(avatar_url, /[\s<>"()\\]/)+ author = escape_text(post.user.username)+ author_link = " [@#{author}](#{post.user.full_url})"+ lines << "### #{I18n.t("markdown_endpoints.post_author", author: author_link)}"+ end+ lines << "#### #{I18n.t("markdown_endpoints.post_date", timestamp: @timestamp.render(post.created_at, url: post.full_url))}"+ lines.concat(["", "</div>", ""])+ lines << cached_body(cooked)+ lines.join("\n")+ end++ def cached_body(cooked)+ context = [+ CookedProcessor::VERSION,+ Discourse.base_url,+ Discourse.asset_host,+ I18n.locale,+ cooked,+ ].join("\0")+ digest = Digest::SHA256.hexdigest(context)+ Discourse+ .cache+ .fetch("markdown-endpoint:cooked:#{digest}", expires_in: 1.week) do+ CookedProcessor.to_markdown(cooked)+ end+ end++ def escape_text(text)+ text+ .to_s+ .gsub(/[\\`*_\[\]<>]/) { |character| "\\#{character}" }+ .sub(/\A[#>-]/) { |character| "\\#{character}" }+ end+ end+enddiff --git a/lib/markdown_endpoint/vary_middleware.rb b/lib/markdown_endpoint/vary_middleware.rbnew file mode 100644index 00000000000..5fc773f9efb--- /dev/null+++ b/lib/markdown_endpoint/vary_middleware.rb@@ -0,0 +1,22 @@+# frozen_string_literal: true++module MarkdownEndpoint+ class VaryMiddleware+ def initialize(app)+ @app = app+ end++ def call(env)+ status, headers, body = @app.call(env)+ return status, headers, body unless env[RequestConstraint::VARY_ACCEPT_ENV_KEY]++ headers = headers.dup+ vary_key = headers.keys.find { |key| key.casecmp?("Vary") } || "Vary"+ tokens = headers[vary_key].to_s.split(",").map(&:strip).reject(&:blank?)+ tokens << "Accept" unless tokens.any? { |token| token.casecmp?("Accept") }+ headers[vary_key] = tokens.join(", ")++ [status, headers, body]+ end+ end+enddiff --git a/lib/topic_list_responder.rb b/lib/topic_list_responder.rbindex eeb8a393bf2..5e0271eebb0 100644--- a/lib/topic_list_responder.rb+++ b/lib/topic_list_responder.rb@@ -16,6 +16,28 @@ module TopicListResponderrender "list/list"endformat.json { render_serialized(list, TopicListSerializer) }+ format.md do+ localize_topic_list_content(list)+ title =+ if @category+ @category.name+ elsif @tag_name+ "##{@tag_name}"+ else+ action_name.titleize+ end+ render_markdown(+ MarkdownEndpoint::TopicListRenderer.new(+ topics: list.topics,+ user: current_user,+ title: title,+ url: markdown_alternate_url,+ page: params[:page],+ next_page_url: (list.more_topics_url if list.topics.size == list.per_page),+ previous_page_url: (list.prev_topics_url if params[:page].to_i.positive?),+ ).render,+ )+ endendenddiff --git a/spec/integrity/middleware_order_spec.rb b/spec/integrity/middleware_order_spec.rbindex 9c5425d921c..7d8ad9eda50 100644--- a/spec/integrity/middleware_order_spec.rb+++ b/spec/integrity/middleware_order_spec.rb@@ -3,6 +3,7 @@RSpec.describe "Middleware order" dolet(:expected_middlewares) do[+ MarkdownEndpoint::VaryMiddleware,BlockRequestsMiddleware,TestMultisiteMiddleware,Middleware::ProcessingRequest,diff --git a/spec/lib/markdown_endpoint/accept_header_spec.rb b/spec/lib/markdown_endpoint/accept_header_spec.rbnew file mode 100644index 00000000000..ae3ce414f17--- /dev/null+++ b/spec/lib/markdown_endpoint/accept_header_spec.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++require "rails_helper"++RSpec.describe MarkdownEndpoint::AcceptHeader do+ describe ".preferred?" do+ it "uses specificity, quality, and the historical HTML tie-break" do+ expect(described_class.preferred?("text/markdown")).to eq(true)+ expect(described_class.preferred?("text/markdown, text/html")).to eq(false)+ expect(described_class.preferred?("text/markdown;q=0, text/html;q=0.1")).to eq(false)+ expect(described_class.preferred?("text/markdown;q=0.8, application/json;q=0.9")).to eq(false)+ expect(described_class.preferred?("text/markdown;q=0.8, text/*;q=0.9")).to eq(false)+ expect(described_class.preferred?("text/markdown;q=0.8, text/html;q=0.7")).to eq(true)+ expect(described_class.preferred?("*/*")).to eq(false)+ expect(described_class.preferred?(nil)).to eq(false)+ end+ end+enddiff --git a/spec/lib/markdown_endpoint/cooked_processor_spec.rb b/spec/lib/markdown_endpoint/cooked_processor_spec.rbnew file mode 100644index 00000000000..8e5ced38cad--- /dev/null+++ b/spec/lib/markdown_endpoint/cooked_processor_spec.rb@@ -0,0 +1,80 @@+# frozen_string_literal: true++require "rails_helper"++RSpec.describe MarkdownEndpoint::CookedProcessor do+ describe ".to_markdown" do+ it "converts GFM structures and Discourse elements" do+ html = <<~HTML+ <h2>Heading</h2>+ <p>Hello <strong>world</strong> <a class="mention" href="/u/sam">@sam</a>+ <a class="hashtag-cooked" href="/tag/ruby">#ruby</a>+ <img class="emoji" title=":wave:" src="/images/emoji.png"></p>+ <table><thead><tr><th>Name</th><th>Value</th></tr></thead><tbody><tr><td>A</td><td>1</td></tr></tbody></table>+ <pre><code class="lang-ruby">puts "ok"</code></pre>+ <details><summary>More</summary><p>Nested <em>content</em></p></details>+ <div class="poll" data-poll-title="Choose one"></div>+ HTML++ markdown = described_class.to_markdown(html)++ expect(markdown).to include("## Heading", "**world**", "@sam", "#ruby", ":wave:")+ expect(markdown).to include("| Name | Value |", 'puts "ok"')+ expect(markdown).to include("> **More**", "_Poll: Choose one (view on site)_")+ end++ it "preserves quotes, oneboxes, lightboxes, and nested formatting" do+ html = <<~HTML+ <aside class="quote" data-username="alice">+ <div class="title"><a href="/t/topic/1/2">alice:</a></div>+ <blockquote><p>A <strong>nested</strong> quote</p></blockquote>+ </aside>+ <aside class="onebox"><h3><a href="https://example.com/x">Example [title]</a></h3><article class="onebox-body"><p>An excerpt</p></article></aside>+ <a class="lightbox" href="/uploads/original.png"><img alt="alt text" src="/uploads/small.png"></a>+ HTML++ markdown = described_class.to_markdown(html)++ expect(markdown).to include("> [@alice](#{Discourse.base_url}/t/topic/1/2):")+ expect(markdown).to include("> A **nested** quote")+ expect(markdown).to include("> **[Example \\[title\\]](https://example.com/x)**")+ expect(markdown).to include("")+ end++ it "escapes every line of plain onebox excerpts" do+ html = <<~HTML+ <aside class="onebox">+ <h3><a href="https://example.test/source">Source</a></h3>+ <article class="onebox-body"><p># Heading<br>[click](https://example.test)</p></article>+ </aside>+ HTML++ markdown = described_class.to_markdown(html)++ expect(markdown).to include("> \\# Heading", "> \\[click\\](https://example.test)")+ expect(markdown).not_to include("\n> # Heading", "\n> [click](https://example.test)")+ end++ it "uses longer code fences locally while preserving language and nested blocks" do+ html = <<~HTML+ <pre><code class="lang-ruby">puts "``` and ````"+ <literal-tag></code></pre>+ <aside class="quote" data-username="alice">+ <blockquote><pre><code class="language-ruby">inside ``` quote</code></pre></blockquote>+ </aside>+ <details><summary>Code details</summary><pre><code>inside ```` details</code></pre></details>+ HTML++ markdown = described_class.to_markdown(html)++ expect(markdown).to include("`````ruby", 'puts "``` and ````"', "<literal-tag>")+ expect(markdown).to include("> ````ruby", "> inside ``` quote", "> `````")+ expect(markdown).to include("> **Code details**", "> `````", "> inside ```` details")+ end++ it "handles malformed and empty HTML without raising" do+ expect(described_class.to_markdown("<p>unfinished <strong>tag")).to include("unfinished")+ expect(described_class.to_markdown(nil)).to eq("")+ end+ end+enddiff --git a/spec/lib/markdown_endpoint/vary_middleware_spec.rb b/spec/lib/markdown_endpoint/vary_middleware_spec.rbnew file mode 100644index 00000000000..4f9a430e919--- /dev/null+++ b/spec/lib/markdown_endpoint/vary_middleware_spec.rb@@ -0,0 +1,19 @@+# frozen_string_literal: true++require "rails_helper"++RSpec.describe MarkdownEndpoint::VaryMiddleware do+ it "preserves Vary tokens and handles Accept case-insensitively" do+ app = ->(_env) { [301, { "vArY" => "Origin, aCcEpT" }, []] }+ env = { MarkdownEndpoint::RequestConstraint::VARY_ACCEPT_ENV_KEY => true }++ _status, headers, _body = described_class.new(app).call(env)++ expect(headers).to eq("vArY" => "Origin, aCcEpT")++ app = ->(_env) { [404, { "Vary" => "Origin" }, []] }+ _status, headers, _body = described_class.new(app).call(env)++ expect(headers).to eq("Vary" => "Origin, Accept")+ end+enddiff --git a/spec/requests/markdown_directories_spec.rb b/spec/requests/markdown_directories_spec.rbnew file mode 100644index 00000000000..8fb8b40151f--- /dev/null+++ b/spec/requests/markdown_directories_spec.rb@@ -0,0 +1,221 @@+# frozen_string_literal: true++require "rails_helper"++RSpec.describe "Markdown directories" do+ fab!(:category) { Fabricate(:category, description: "<p>A useful category.</p>") }+ fab!(:child) { Fabricate(:category, parent_category: category) }+ fab!(:tag) do+ Fabricate(:tag, public_topic_count: 1, staff_topic_count: 1, description: "A useful tag.")+ end++ before do+ SiteSetting.experimental_markdown_endpoints = true+ SiteSetting.tagging_enabled = true+ end++ after { category.clear_url_cache }++ it "serves both directories by suffix and negotiation, with discovery on HTML" do+ %w[categories tags].each do |directory|+ get "/#{directory}.md", headers: { "ACCEPT" => "application/json" }+ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/markdown")++ get "/#{directory}", headers: { "ACCEPT" => "text/markdown" }+ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/markdown")+ expect(response.headers["Vary"]).to include("Accept")++ get "/#{directory}", headers: { "ACCEPT" => "text/html" }+ expect(response.media_type).to eq("text/html")+ expect(response.headers["Link"]).to include("/#{directory}.md")+ expect(response.body).to include('type="text/markdown"', "/#{directory}.md")++ get "/#{directory}.json", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).to eq("application/json")+ end+ end++ it "renders category descriptions, children, and parent links" do+ get "/categories.md?include_subcategories=true"++ expect(response.body).to include(+ "[#{category.name}](#{Discourse.base_url_no_prefix}#{category.url}.md)",+ "[#{child.name}](#{Discourse.base_url_no_prefix}#{child.url}.md)",+ "Parent: [#{category.name}]",+ "A useful category.",+ )+ expect(response.body).not_to include("<p>")+ end++ it "uses category permissions for anonymous and authorized readers" do+ restricted = Fabricate(:category, read_restricted: true)+ restricted.set_permissions(staff: :full)++ get "/categories.md"+ expect(response.body).not_to include(restricted.name)++ sign_in(Fabricate(:admin))+ get "/categories.md"+ expect(response.body).to include(restricted.name)+ end++ it "uses the native tag visibility rules for anonymous readers and admins" do+ restricted = Fabricate(:tag, public_topic_count: 1, staff_topic_count: 1)+ Fabricate(:tag_group, permissions: { "staff" => 1 }, tag_names: [restricted.name])+ pm_only = Fabricate(:tag, public_topic_count: 0, staff_topic_count: 0, pm_topic_count: 1)+ synonym = Fabricate(:tag, target_tag: tag, public_topic_count: 1)++ get "/tags.md"+ expect(response.body).to include(+ "[#{tag.name}](#{Discourse.base_url_no_prefix}#{tag.url}.md)",+ tag.description,+ )+ expect(response.body).not_to include(restricted.name, pm_only.name, synonym.name)++ sign_in(Fabricate(:admin))+ get "/tags.md"+ expect(response.body).to include(restricted.name)+ expect(response.body).to include(pm_only.name)+ expect(response.body).not_to include(synonym.name)+ end++ it "includes grouped tags and deduplicates category-associated tags" do+ grouped = Fabricate(:tag, public_topic_count: 1)+ Fabricate(:tag_group, tag_names: [grouped.name])+ category.tags << tag++ [false, true].each do |grouped_layout|+ SiteSetting.tags_listed_by_group = grouped_layout+ get "/tags.md"+ expect(response.body.scan(/^## \[(.*?)\]/).flatten).to contain_exactly(tag.name, grouped.name)+ end+ end++ it "uses the native category ordering and pagination" do+ sign_in(Fabricate(:user))+ SiteSetting.lazy_load_categories_groups = "#{Group::AUTO_GROUPS[:everyone]}"+ stub_const(CategoryList, :CATEGORIES_PER_PAGE, 2) do+ [1, 2].each do |page|+ get "/categories.json", params: { page: page }+ names = response.parsed_body["category_list"]["categories"].map { |entry| entry["name"] }++ get "/categories.md", params: { page: page }+ expect(response.body.scan(/^## \[(.*?)\]/).flatten).to eq(names)+ end+ end+ end++ it "respects the native category layout's subcategory selection" do+ SiteSetting.desktop_category_page_style = "categories_only"+ SiteSetting.mobile_category_page_style = "categories_only"+ get "/categories.md"+ expect(response.body).to include(category.name)+ expect(response.body).not_to include(child.name)+ end++ it "omits next-page links when category pagination is disabled" do+ SiteSetting.lazy_load_categories_groups = ""+ stub_const(CategoryList, :CATEGORIES_PER_PAGE, 2) do+ get "/categories.md"+ expect(response.body.scan(/^## /).size).to be >= 2+ expect(response.body).not_to include("[Next page]")+ end+ end++ it "links only to existing native category pages, without counting appended children" do+ sign_in(Fabricate(:user))+ SiteSetting.lazy_load_categories_groups = "#{Group::AUTO_GROUPS[:everyone]}"+ SiteSetting.fixed_category_positions = true+ category.update!(position: 1000)+ Fabricate(:category, parent_category: category)++ stub_const(CategoryList, :CATEGORIES_PER_PAGE, 2) do+ url = "/categories.md"+ 10.times do+ get url+ expect(response.body).to include("## [")+ next_url = response.body[/\[Next page\]\(([^)]+)\)/, 1]+ break unless next_url+ url = URI(next_url).request_uri+ end+ expect(response.body).not_to include("[Next page]")+ expect(response.body).to include(category.name, child.name)+ end+ end++ it "renders localized directory entries with localization preloading" do+ SiteSetting.content_localization_enabled = true+ category.update!(locale: "ja")+ child.update!(locale: "ja")+ tag.update!(locale: "ja")+ Fabricate(+ :category_localization,+ category: category,+ locale: "en",+ name: "Translated category",+ description: "<p>Translated description.</p>",+ )+ Fabricate(:category_localization, category: child, locale: "en", name: "Translated child")+ Fabricate(+ :tag_localization,+ tag: tag,+ locale: "en",+ name: "translated-tag",+ description: "Translated tag description.",+ )++ queries = track_sql_queries { get "/categories.md?include_subcategories=true" }+ expect(response.body).to include(+ "Translated category",+ "Translated child",+ "Translated description.",+ "Parent: [Translated category]",+ )+ expect(queries.grep(/FROM "category_localizations"/).length).to be <= 2++ get "/tags.md"+ expect(response.body).to include("translated-tag", "Translated tag description.")+ end++ it "uses subfolder-safe links in directories and topic-list navigation" do+ set_subfolder "/forum"+ get "/latest.md"+ expect(response.body).to include(+ "\n[Latest](#{Discourse.base_url}/latest.md) · [Categories](#{Discourse.base_url}/categories.md) · [Tags](#{Discourse.base_url}/tags.md)\n",+ )++ get "/categories.md?include_subcategories=true"+ expect(response.body).to include("#{Discourse.base_url_no_prefix}#{child.url}.md")+ expect(response.body).not_to include("/forum/forum/")+ end++ it "omits the tags link and rejects the directory when tagging is disabled" do+ SiteSetting.tagging_enabled = false+ get "/latest.md"+ expect(response.body).to include("/categories.md")+ expect(response.body).not_to include("/tags.md")++ get "/tags.md"+ expect(response).to have_http_status(:not_found)+ end++ it "renders an empty tag directory without a next page" do+ tag.update!(public_topic_count: 0)+ get "/tags.md"+ expect(response).to have_http_status(:ok)+ expect(response.body).to include("No entries to display.")+ expect(response.body).not_to include("[Next page]")+ end++ it "keeps directories behind the Markdown feature setting" do+ SiteSetting.experimental_markdown_endpoints = false+ %w[categories tags].each do |directory|+ get "/#{directory}.md"+ expect(response).to have_http_status(:not_found)+ get "/#{directory}", headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to be_blank+ end+ end+enddiff --git a/spec/requests/markdown_endpoints_spec.rb b/spec/requests/markdown_endpoints_spec.rbnew file mode 100644index 00000000000..e3a5ff0d3f9--- /dev/null+++ b/spec/requests/markdown_endpoints_spec.rb@@ -0,0 +1,851 @@+# frozen_string_literal: true++require "rails_helper"++RSpec.describe "Markdown endpoints" do+ fab!(:user) { Fabricate(:user, trust_level: TrustLevel[2]) }+ fab!(:category)+ fab!(:topic) do+ Fabricate(:topic, user: user, category: category, title: "Markdown endpoint topic")+ end+ fab!(:post) { Fabricate(:post, topic: topic, user: user, raw: "Visible body") }++ before { SiteSetting.experimental_markdown_endpoints = true }+ after { category.clear_url_cache }++ it "keeps HTML tag-intersection redirects out of Markdown routes" do+ tag = Fabricate(:tag)++ [false, true].each do |enabled|+ SiteSetting.experimental_markdown_endpoints = enabled+ get "/tags/intersection/#{tag.name}/#{tag.name}", headers: { "ACCEPT" => "text/html" }+ expect(response).to redirect_to("/tag/#{tag.name}")+ follow_redirect!+ expect(response).to redirect_to(tag.url)+ follow_redirect!+ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/html")+ end+ end++ def vary_tokens+ response.headers.fetch("Vary", "").split(",").map(&:strip)+ end++ it "serves latest topics for Markdown requests to the homepage" do+ get "/", headers: { "ACCEPT" => "text/markdown" }++ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/markdown")+ expect(response.body).to include(topic.title, "**URL:** #{Discourse.base_url}/latest.md")+ expect(vary_tokens).to include("Accept")++ head "/", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).to eq("text/markdown")+ expect(response.body).to be_empty++ get "/.md"+ expect(response).to have_http_status(:not_found)+ end++ it "preserves the configured HTML homepage and advertises latest Markdown" do+ Fabricate(:admin)+ SiteSetting.top_menu = "categories|latest|hot|top"++ [+ "text/html",+ "*/*",+ "text/markdown;q=0, text/html",+ "text/markdown, text/html",+ ].each do |accept|+ get "/", headers: { "ACCEPT" => accept }++ expect(response).to have_http_status(:ok), accept+ expect(response.media_type).to eq("text/html")+ expect(controller.controller_name).to eq("categories")+ expect(response.headers["Link"]).to include("#{Discourse.base_url}/latest.md")+ expect(response.body).to include('type="text/markdown"', "/latest.md")+ expect(vary_tokens).to include("Accept")+ end++ get "/", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).to eq("text/markdown")+ expect(response.body).to start_with("# Latest")+ end++ it "adds Vary: Accept to negotiated responses and redirects" do+ [+ %w[text/html text/html],+ %w[application/json application/json],+ %w[text/markdown text/markdown],+ ].each do |accept, media_type|+ get "/t/#{topic.slug}/#{topic.id}", headers: { "ACCEPT" => accept }++ expect(response.media_type).to eq(media_type)+ expect(vary_tokens.grep(/\Aaccept\z/i).length).to eq(1)+ end++ get "/t/#{topic.id}", headers: { "ACCEPT" => "text/markdown" }+ expect(response).to have_http_status(:moved_permanently)+ expect(vary_tokens.grep(/\Aaccept\z/i).length).to eq(1)++ get "/t/#{topic.slug}/#{topic.id}/999", headers: { "ACCEPT" => "text/markdown" }+ expect(response).to have_http_status(:not_found)+ expect(vary_tokens.grep(/\Aaccept\z/i).length).to eq(1)+ end++ it "preserves explicit Markdown when redirecting an out-of-range topic page" do+ 2.upto(TopicView::CHUNK_SIZE + 1) do |post_number|+ Fabricate(:post, topic:, user:, post_number:)+ end++ get "/t/#{topic.slug}/#{topic.id}.md?page=999"+ expect(response).to have_http_status(:moved_permanently)+ expect(response.location).to end_with(".md?page=2")+ follow_redirect!+ expect(response.media_type).to eq("text/markdown")++ get "/t/#{topic.slug}/#{topic.id}.json?page=999"+ expect(response.location).to end_with(".json?page=2")++ get "/t/#{topic.slug}/#{topic.id}?page=999", headers: { "ACCEPT" => "text/html" }+ expect(response.location).to end_with("/#{topic.id}?page=2")+ end++ it "supports suffix and Accept negotiation without overriding HTML, JSON, or RSS" do+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/markdown")+ expect(response.body).to include("# Markdown endpoint topic", post.raw)+ expect(response.headers["Vary"].split(",").map(&:strip)).to include("Accept")++ get "/t/#{topic.slug}/#{topic.id}",+ headers: {+ "ACCEPT" => "text/markdown;q=0.9, text/html;q=0.8",+ }+ expect(response.media_type).to eq("text/markdown")++ get "/t/#{topic.id}", headers: { "ACCEPT" => "text/markdown" }+ expect(response.location).to end_with(".md")+ follow_redirect!+ expect(response.media_type).to eq("text/markdown")++ get "/t/#{topic.slug}/#{topic.id}", headers: { "ACCEPT" => "text/markdown, text/html" }+ expect(response.media_type).to eq("text/html")++ get "/t/#{topic.slug}/#{topic.id}",+ headers: {+ "ACCEPT" => "text/markdown;q=0.8, application/json;q=0.9",+ }+ expect(response.media_type).to eq("application/json")++ get "/t/#{topic.slug}/#{topic.id}.json", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).to eq("application/json")++ get "/t/#{topic.slug}/#{topic.id}.rss", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).to eq("application/rss+xml")+ end++ it "preserves explicit formats and unrelated topic and category routes" do+ tag = Fabricate(:tag)+ topic.tags << tag++ [+ "/t/#{topic.slug}/#{topic.id}.json",+ "/t/#{topic.slug}/#{topic.id}.rss",+ "/c/#{category.slug}/#{category.id}.json",+ "/c/#{category.slug}/#{category.id}.rss",+ "/tag/#{tag.slug_for_url}/#{tag.id}.json",+ "/tag/#{tag.slug_for_url}/#{tag.id}.rss",+ "/t/#{topic.id}/posts",+ "/t/#{topic.id}/post_ids",+ ].each do |path|+ get path, headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).not_to eq("text/markdown"), path+ end++ get "/about", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).not_to eq("text/markdown")++ get "/c/#{category.slug}/#{category.id}/subcategories", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).not_to eq("text/markdown")+ end++ it "limits Markdown and discovery to the supported routes" do+ tag = Fabricate(:tag)+ paths = [+ "/unseen",+ "/bookmarks",+ "/top/yearly",+ "/c/#{category.slug}/#{category.id}/none",+ "/c/#{category.slug}/#{category.id}/l/latest",+ "/tag/#{tag.slug_for_url}/#{tag.id}/l/latest",+ "/u/#{user.username}/activity",+ ]++ paths.each do |path|+ get "#{path}.md"+ expect(response).to have_http_status(:not_found), path++ get path, headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).not_to eq("text/markdown"), path+ expect(response.headers["Link"]).to be_blank, path++ get path, headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to be_blank, path+ expect(response.body).not_to include('type="text/markdown"'), path+ end+ end++ it "lets the explicit suffix win and handles HEAD and invalid posts" do+ get "/t/#{topic.slug}/#{topic.id}.md", headers: { "ACCEPT" => "application/json" }+ expect(response.media_type).to eq("text/markdown")++ head "/t/#{topic.slug}/#{topic.id}.md"+ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/markdown")+ expect(response.body).to be_empty++ head "/latest.md"+ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/markdown")+ expect(response.body).to be_empty++ get "/t/#{topic.slug}/#{topic.id}/999.md"+ expect(response).to have_http_status(:not_found)+ end++ it "paginates rendered posts with translated Markdown navigation links" do+ TranslationOverride.upsert!("en", "markdown_endpoints.previous_page", "Earlier posts")+ TranslationOverride.upsert!("en", "markdown_endpoints.next_page", "Later posts")+ replies =+ 2+ .upto(TopicView::CHUNK_SIZE + 1)+ .map do |post_number|+ Fabricate(:post, topic:, user:, post_number:, raw: "Reply #{post_number}")+ end++ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include(+ post.raw,+ "**Page:** 1",+ "[Later posts](#{topic.url}.md?page=2)",+ )+ expect(response.body).not_to include(replies.last.raw)++ get "/t/#{topic.slug}/#{topic.id}.md?page=2"+ expect(response.body).to include(+ replies.last.raw,+ "**Page:** 2",+ "[Earlier posts](#{topic.url}.md?page=1)",+ )+ expect(response.body).not_to include(post.raw, "[Later posts]")+ end++ it "renders just the requested post" do+ reply = Fabricate(:post, topic:, user:, raw: "Reply body")++ get "/t/#{topic.slug}/#{topic.id}/#{reply.post_number}.md"+ expect(response.body).to include(+ reply.raw,+ "**Showing post:** #{reply.post_number}",+ "View the full topic",+ )+ expect(response.body).not_to include(post.raw, "[Next page]", "[Previous page]")+ end++ it "links topic and post authors to subfolder-safe user profiles" do+ set_subfolder "/forum"+ SiteSetting.external_system_avatars_url = ""+ author = Fabricate(:user, username: "reply_author")+ reply = Fabricate(:post, topic: topic, user: author)++ get "/latest.md"+ expect(response.body).to include(+ "**Author:** [@#{user.username}](#{Discourse.base_url}/u/#{user.encoded_username})",+ )+ expect(response.body).not_to include("",+ "### Author:  [@reply\\_author](#{Discourse.base_url}/u/#{author.encoded_username})",+ "\n#### Post date: ",+ )++ get "/t/#{topic.slug}/#{topic.id}/#{reply.post_number}.md"+ expect(response.body).to include(+ "[@reply\\_author](#{Discourse.base_url}/u/#{author.encoded_username})",+ )++ reply.update_columns(user_id: nil)+ get "/t/#{topic.slug}/#{topic.id}/#{reply.post_number}.md"+ expect(response).to have_http_status(:ok)+ expect(response.body).to include("#### Post date: ")+ expect(response.body).not_to include("### Author:")+ end++ it "wraps list metadata separately from titles and excerpts with Markdown block boundaries" do+ topic.update!(excerpt: "An excerpt outside metadata", last_posted_at: post.created_at)++ get "/latest.md"++ metadata = response.body.scan(%r{<div class="topic-metadata">\n\n(.*?)\n\n</div>}m)+ expect(metadata.size).to eq(1)+ expect(metadata.first.first).to include("**Author:**", "**Replies:**", "**Last updated:**")+ expect(metadata.first.first).to include("\\\n**Replies:**", "\\\n**Last updated:**")+ expect(metadata.first.first).not_to include(topic.title, topic.excerpt)+ expect(response.body).to include(+ "## [#{topic.title}](#{topic.url})\n\n<div class=\"topic-metadata\">",+ "</div>\n\n#{topic.excerpt}",+ )+ end++ it "wraps each post's headings separately from its body with Markdown block boundaries" do+ reply = Fabricate(:post, topic: topic, user: user, raw: "Another visible body")++ [+ "/t/#{topic.slug}/#{topic.id}.md",+ "/t/#{topic.slug}/#{topic.id}/#{reply.post_number}.md",+ ].each do |path|+ get path++ posts = path.end_with?("/#{reply.post_number}.md") ? [reply] : [post, reply]+ metadata = response.body.scan(%r{<div class="post-metadata">\n\n(.*?)\n\n</div>}m).flatten+ expect(metadata.size).to eq(posts.size)+ posts+ .zip(metadata)+ .each do |rendered_post, block|+ expect(block).to include("### Author: ",+ )+ end++ it "formats timestamps in the reader's timezone with precise link titles and list reply counts" do+ timestamp = Time.utc(2026, 7, 15, 16, 30, 45)+ post.update_columns(created_at: timestamp)+ topic.update_columns(last_posted_at: timestamp, posts_count: 3)+ user.user_option.update!(timezone: "America/Toronto")+ sign_in(user)++ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include(+ %(#### Post date: [July 15, 2026, 12:30pm EDT](#{post.full_url} "2026-07-15T12:30:45-04:00")),+ )++ get "/latest.md"+ expect(response.body).to include(+ "**Replies:** 2",+ %(**Last updated:** [July 15, 2026, 12:30pm EDT](#{topic.url} "2026-07-15T12:30:45-04:00")),+ )+ end++ it "uses the application timezone for anonymous timestamps" do+ post.update_columns(created_at: Time.utc(2026, 1, 15, 16, 30, 45))++ Time.use_zone("America/New_York") do+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include(+ %([January 15, 2026, 11:30am EST](#{post.full_url} "2026-01-15T11:30:45-05:00")),+ )+ end++ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include(+ %([January 15, 2026, 4:30pm UTC](#{post.full_url} "2026-01-15T16:30:45Z")),+ )++ user.user_option.update!(timezone: nil)+ sign_in(user)+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include(+ %([January 15, 2026, 4:30pm UTC](#{post.full_url} "2026-01-15T16:30:45Z")),+ )+ end++ it "preserves topic author filters through discovery and pagination" do+ excluded = Fabricate(:post, topic: topic, raw: "Another author's reply")+ 2.times { Fabricate(:post, topic: topic, user: user) }++ [{ username_filters: user.username }, { replies_to_post_number: "1" }].each do |filters|+ get "/t/#{topic.slug}/#{topic.id}", params: filters, headers: { "ACCEPT" => "text/html" }+ alternate = response.headers.fetch("Link")[/<([^>]+)>/, 1]+ expect(Rack::Utils.parse_nested_query(URI(alternate).query)).to eq(filters.stringify_keys)+ end++ stub_const(TopicView, :CHUNK_SIZE, 2) do+ get "/t/#{topic.slug}/#{topic.id}.md", params: { username_filters: user.username }+ next_url = response.body[/\[Next page\]\(([^)]+)\)/, 1]+ expect(Rack::Utils.parse_nested_query(URI(next_url).query)).to eq(+ "page" => "2",+ "username_filters" => user.username,+ )+ get URI(next_url).request_uri+ expect(response.body).not_to include(excluded.raw)+ expect(response.body).to include("username_filters=#{user.username}")+ end+ end++ it "invalidates transformed bodies after cooked-only changes in the same second" do+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include(post.raw)++ post.update_columns(cooked: "<p>Rebaked body</p>")+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include("Rebaked body")+ expect(response.body).not_to include(post.raw)+ end++ it "renders cooked content with portable links instead of raw authoring syntax" do+ post.update_columns(+ raw: "[details=More]Original source[/details]",+ cooked:+ '<details><summary>More</summary><p>Rendered <strong>content</strong> <a href="/latest">topics</a></p></details>',+ )++ get "/t/#{topic.slug}/#{topic.id}.md"++ expect(response.body).to include(+ "> **More**",+ "Rendered **content**",+ "[topics](#{Discourse.base_url}/latest)",+ )+ expect(response.body).not_to include(post.raw, "<details>")+ end++ it "uses subfolder-safe topic links in lists and discovery" do+ set_subfolder "/forum"++ get "/latest.md"+ expect(response.body).to include("#{Discourse.base_url}/t/#{topic.slug}/#{topic.id}")+ expect(response.body).not_to include("/forum/forum/")++ get "/latest", headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to include("#{Discourse.base_url}/latest.md")+ expect(response.headers["Link"]).not_to include("/forum/forum/")+ end++ it "preloads distinct topic authors while rendering lists" do+ 3.times do+ author = Fabricate(:user)+ listed_topic = Fabricate(:topic, user: author)+ Fabricate(:post, topic: listed_topic, user: author)+ end++ queries = track_sql_queries { get "/latest.md" }+ individual_author_queries =+ queries.grep(/FROM "users" WHERE "users"\."id" = \d+ (?:ORDER BY .+ )?LIMIT 1/)++ expect(response).to have_http_status(:ok)+ expect(individual_author_queries).to be_empty+ end++ it "preloads and renders localized titles and post bodies" do+ SiteSetting.content_localization_enabled = true+ topic.update!(locale: "ja")+ Fabricate(:topic_localization, topic:, locale: "en", title: "Translated topic title")+ post.update!(locale: "ja")+ Fabricate(:post_localization, post:, locale: "en", cooked: "<p>Translated post 1</p>")+ 2.upto(3) do |post_number|+ localized_post =+ Fabricate(:post, topic:, user:, post_number:, locale: "ja", raw: "Original #{post_number}")+ Fabricate(+ :post_localization,+ post: localized_post,+ locale: "en",+ cooked: "<p>Translated post #{post_number}</p>",+ )+ end++ queries = track_sql_queries { get "/t/#{topic.slug}/#{topic.id}.md" }++ expect(response.body).to include("# Translated topic title", "Translated post 3")+ expect(queries.grep(/FROM "post_localizations"/).length).to be <= 2+ end++ it "invalidates transformed bodies when the asset host changes" do+ post.update_columns(cooked: '<p><img src="/uploads/example.png" alt="Example"></p>')++ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include("#{Discourse.base_url}/uploads/example.png")++ set_cdn_url "https://cdn.example.test"+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include("https://cdn.example.test/uploads/example.png")+ end++ it "serves the main topic lists by suffix and Accept header" do+ tag = Fabricate(:tag, name: "markdown-tag")+ topic.tags << tag+ subcategory = Fabricate(:category, parent_category: category)+ paths = [+ "/latest",+ "/hot",+ "/top",+ "/c/#{category.slug}/#{category.id}",+ "/c/#{category.slug}/#{subcategory.slug}/#{subcategory.id}",+ "/tag/#{tag.name}",+ "/tag/#{tag.slug_for_url}/#{tag.id}",+ ]++ paths.each do |path|+ ["#{path}.md", path].each do |url|+ get url, headers: { "ACCEPT" => "text/markdown" }+ if response.moved_permanently?+ expect(response.location).to include(".md")+ follow_redirect!+ end+ expect(response).to have_http_status(:ok), "#{url}: #{response.status}"+ expect(response.media_type).to eq("text/markdown"), url+ end+ end+ end++ it "serves personalized new and unread lists with discovery and native filtering" do+ SiteSetting.enable_unified_new = true+ new_topic = Fabricate(:post).topic+ unread_topic = Fabricate(:new_reply_topic, current_user: user)+ read_topic = Fabricate(:read_topic, current_user: user)+ sign_in(user)++ { "/new" => new_topic, "/unread" => unread_topic }.each do |path, expected_topic|+ get "#{path}.json"+ expected_ids =+ response.parsed_body.fetch("topic_list").fetch("topics").map { |entry| entry.fetch("id") }+ expect(expected_ids).to include(expected_topic.id)++ ["#{path}.md", path].each do |url|+ get url, headers: { "ACCEPT" => "text/markdown" }+ expect(response).to have_http_status(:ok)+ expect(response.media_type).to eq("text/markdown")+ ids = response.body.scan(%r{^## \[.*\]\([^\n]*/t/[^\n]+/(\d+)\)$}).flatten.map(&:to_i)+ expect(ids).to match_array(expected_ids)+ expect(response.body).not_to include(read_topic.title)+ end++ get path, headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to include(+ "<#{Discourse.base_url}#{path}.md>; rel=\"alternate\"",+ )+ expect(response.body).to include('type="text/markdown"')+ end++ get "/new.md", params: { subset: "replies" }+ expect(response.body).to include(unread_topic.title)+ expect(response.body).not_to include(new_topic.title)+ end++ it "requires authentication for new and unread Markdown lists" do+ %w[/new /unread].each do |path|+ ["#{path}.md", path].each do |url|+ get url, headers: { "ACCEPT" => "text/markdown" }+ expect(response).to have_http_status(:not_found)+ expect(response.media_type).not_to eq("text/markdown")+ end+ end+ end++ it "paginates personalized lists while preserving subset filters" do+ SiteSetting.enable_unified_new = true+ 2.times { Fabricate(:new_reply_topic, current_user: user) }+ sign_in(user)++ %w[/new /unread].each do |path|+ get "#{path}.md", params: { per_page: 1, subset: "replies" }+ first_title = response.body[/^## \[(.*?)\]/, 1]+ next_url = response.body[/\[Next page\]\(([^)]+)\)/, 1]+ expect(next_url).to start_with("#{Discourse.base_url}#{path}.md?")+ expect(Rack::Utils.parse_nested_query(URI(next_url).query)).to include(+ "page" => "1",+ "per_page" => "1",+ "subset" => "replies",+ )++ get URI(next_url).request_uri+ expect(response).to have_http_status(:ok)+ expect(response.body[/^## \[(.*?)\]/, 1]).not_to eq(first_title)+ expect(response.body).to include("[Previous page](#{Discourse.base_url}#{path}.md?")+ end+ end++ it "retains native list pagination" do+ TopicQuery::DEFAULT_PER_PAGE_COUNT.times do |index|+ listed_topic = Fabricate(:topic, user: user, title: "Paginated topic #{index}")+ Fabricate(:post, topic: listed_topic, user: user)+ end++ get "/latest.md?page=1"+ expect(response).to have_http_status(:ok)+ expect(response.body).to include("**Page:** 2")+ expect(response.body).not_to include("Topics on this page", "[Next page]")+ end++ it "follows Markdown list pagination while retaining filters and subfolder paths" do+ set_subfolder "/forum"+ category.clear_url_cache+ other = Fabricate(:topic, user: user, category: category)+ Fabricate(:post, topic: other, user: user)+ tag = Fabricate(:tag)+ [topic, other].each do |listed_topic|+ listed_topic.tags << tag+ TopicHotScore.create!(topic: listed_topic, score: listed_topic.id)+ TopTopic.create!(topic: listed_topic, yearly_score: listed_topic.id)+ end++ [+ "/latest",+ "/hot",+ "/top",+ "/c/#{category.slug}/#{category.id}",+ "/tag/#{tag.slug_for_url}/#{tag.id}",+ ].each do |path|+ get "#{path}.md", params: { per_page: 1, category: category.id, period: "yearly" }+ expect(response).to have_http_status(:ok), "#{path}: #{response.status} #{response.location}"+ first_title = response.body[/^## \[(.*?)\]/, 1]+ next_url = response.body[/\[Next page\]\(([^)]+)\)/, 1]+ expect(next_url).to start_with("#{Discourse.base_url}#{path}.md?")+ expect(Rack::Utils.parse_nested_query(URI(next_url).query)).to include(+ "page" => "1",+ "per_page" => "1",+ )++ get URI(next_url).request_uri.delete_prefix(Discourse.base_path)+ expect(response).to have_http_status(:ok)+ expect(response.body[/^## \[(.*?)\]/, 1]).not_to eq(first_title)+ previous_url = response.body[/\[Previous page\]\(([^)]+)\)/, 1]+ expect(previous_url).to start_with("#{Discourse.base_url}#{path}.md?")+ get URI(previous_url).request_uri.delete_prefix(Discourse.base_path)+ expect(response.body[/^## \[(.*?)\]/, 1]).to eq(first_title)+ end+ end++ it "preserves topic, category, and tag authorization" do+ private_category = Fabricate(:category, read_restricted: true)+ private_category.set_permissions(staff: :full)+ private_topic =+ Fabricate(:topic, category: private_category, user: user, title: "Restricted private topic")+ private_post = Fabricate(:post, topic: private_topic, user: user, raw: "Private body")++ get "/t/#{private_topic.slug}/#{private_topic.id}.md"+ expect(response).to have_http_status(:not_found)++ get "/latest.md"+ expect(response.body).not_to include(private_topic.title, private_post.raw)++ sign_in(Fabricate(:admin))+ get "/t/#{private_topic.slug}/#{private_topic.id}.md"+ expect(response).to have_http_status(:ok)+ expect(response.body).to include(private_post.raw)+ end++ it "preserves personal-message participant checks" do+ recipient = Fabricate(:user)+ private_message = Fabricate(:private_message_topic, user: user, recipient: recipient)+ private_post = Fabricate(:post, topic: private_message, user: user, raw: "Message secret")++ get "/t/#{private_message.slug}/#{private_message.id}.md"+ expect(response).to have_http_status(:not_found)++ sign_in(recipient)+ get "/t/#{private_message.slug}/#{private_message.id}.md"+ expect(response).to have_http_status(:ok)+ expect(response.body).to include(private_post.raw)++ SiteSetting.log_personal_messages_views = true+ admin = Fabricate(:admin)+ sign_in(admin)+ get "/t/#{private_message.slug}/#{private_message.id}.md"+ expect(response).to have_http_status(:ok)+ expect(UserHistory.last).to have_attributes(+ acting_user_id: admin.id,+ action: UserHistory.actions[:check_personal_message],+ topic_id: private_message.id,+ )+ end++ it "filters whispers according to the configured groups, including for admins" do+ whisper_group = Fabricate(:group)+ member = Fabricate(:user)+ whisper_group.add(member)+ SiteSetting.whispers_allowed_groups = whisper_group.id.to_s+ whisper =+ Fabricate(+ :post,+ topic: topic,+ user: member,+ post_type: Post.types[:whisper],+ raw: "Group whisper",+ )++ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).not_to include(whisper.raw)++ sign_in(Fabricate(:admin))+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).not_to include(whisper.raw)++ sign_in(member)+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).to include(whisper.raw)+ end++ it "uses serialized redaction for hidden posts and does not cache it across readers" do+ post.update!(hidden: true)++ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).not_to include(post.raw)+ anonymous_body = response.body++ sign_in(user)+ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response.body).not_to include(post.raw)+ expect(response.body).not_to eq(anonymous_body)+ end++ it "uses visible tags and excludes deleted posts from anonymous output" do+ visible_tag = Fabricate(:tag, name: "visible-markdown-tag")+ restricted_tag = Fabricate(:tag, name: "restricted-markdown-tag")+ Fabricate(:tag_group, permissions: { "staff" => 1 }, tag_names: [restricted_tag.name])+ topic.tags = [visible_tag, restricted_tag]+ deleted_post = Fabricate(:post, topic: topic, user: user, raw: "Deleted body secret")+ deleted_post.trash!++ get "/t/#{topic.slug}/#{topic.id}.md"++ expect(response).to have_http_status(:ok)+ expect(response.body).to include(visible_tag.name)+ expect(response.body).not_to include(restricted_tag.name, deleted_post.raw)+ end++ it "advertises only real supported counterparts in headers, HTML, and feeds" do+ tag = Fabricate(:tag, name: "discovery-tag")+ topic.tags << tag++ get "/latest?page=2&unknown=secret", headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to include("/latest.md?page=2")+ expect(response.headers["Link"]).not_to include("unknown", "secret")+ expect(response.body).to include('rel="alternate"', 'type="text/markdown"', "/latest.md?page=2")++ [+ "/t/#{topic.slug}/#{topic.id}",+ "/hot",+ "/top",+ "/c/#{category.slug}/#{category.id}",+ "/tag/#{tag.slug_for_url}/#{tag.id}",+ ].each do |path|+ get path, headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to include("#{path}.md"), path+ expect(response.body).to include('type="text/markdown"', "#{path}.md"), path+ end++ [+ "/latest.rss?page=2&unknown=secret",+ "/hot.rss",+ "/top.rss?period=yearly",+ "/t/#{topic.slug}/#{topic.id}.rss",+ "/c/#{category.slug}/#{category.id}.rss",+ "/tag/#{tag.slug_for_url}/#{tag.id}.rss",+ ].each do |path|+ get path+ expect(response.body).to include("<atom:link", 'type="text/markdown"'), path+ expect(response.body).not_to include("unknown=secret"), path+ end++ [+ "/about",+ "/topics/created-by/#{user.username}",+ "/u/#{user.username}/activity/topics.rss",+ "/t/#{topic.id}/last",+ "/t/#{topic.id}/summary",+ "/t/#{topic.slug}/#{topic.id}/print",+ ].each do |path|+ get path, headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to be_blank, path+ if response.media_type == "text/html"+ expect(response.body).not_to include('type="text/markdown"'), path+ end+ end+ end++ it "preserves safe filters in discovery and Markdown tag redirects" do+ tag = Fabricate(:tag, name: "filtered-markdown-tag")+ topic.tags << tag+ other_topic = Fabricate(:topic, user:, title: "Unfiltered topic")+ Fabricate(:post, topic: other_topic, user:)+ query =+ Rack::Utils.build_nested_query(+ category: category.id,+ tags: [tag.name],+ per_page: 1,+ match_all_tags: true,+ api_key: "api-secret",+ user_api_key: "user-secret",+ api_username: user.username,+ token: "token-secret",+ unknown: "unknown-secret",+ )++ get "/latest?#{query}", headers: { "ACCEPT" => "text/html" }+ alternate_url = response.headers.fetch("Link")[/<([^>]+)>/, 1]+ alternate_query = Rack::Utils.parse_nested_query(URI.parse(alternate_url).query)+ expect(alternate_query).to eq(+ "category" => category.id.to_s,+ "match_all_tags" => "true",+ "per_page" => "1",+ "tags" => [tag.name],+ )++ uri = URI.parse(alternate_url)+ get "#{uri.path}?#{uri.query}"+ expect(response.body).to include(topic.title)+ expect(response.body).not_to include(other_topic.title)++ get "/tag/#{tag.name}.md?#{query}"+ expect(response).to have_http_status(:moved_permanently)+ redirect_query = Rack::Utils.parse_nested_query(URI.parse(response.location).query)+ expect(redirect_query).to eq(alternate_query)+ end++ it "keeps Markdown routes and discovery absent while disabled" do+ SiteSetting.experimental_markdown_endpoints = false++ sign_in(user)+ %w[/new /unread].each do |path|+ get "#{path}.md"+ expect(response).to have_http_status(:not_found)++ get path, headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).not_to eq("text/markdown")+ expect(response.headers["Link"]).to be_blank+ end++ get "/t/#{topic.slug}/#{topic.id}.md"+ expect(response).to have_http_status(:not_found)++ get "/t/#{topic.slug}/#{topic.id}", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).not_to eq("text/markdown")++ get "/latest", headers: { "ACCEPT" => "text/html" }+ expect(response.headers["Link"]).to be_blank+ expect(response.body).not_to include('type="text/markdown"')++ get "/", headers: { "ACCEPT" => "text/markdown" }+ expect(response.media_type).not_to eq("text/markdown")++ get "/", headers: { "ACCEPT" => "text/html" }+ expect(response.media_type).to eq("text/html")+ expect(response.headers["Link"]).to be_blank+ end+enddiff --git a/spec/requests/tags_controller_spec.rb b/spec/requests/tags_controller_spec.rbindex fd8f1e0f230..98a41ee9b3c 100644--- a/spec/requests/tags_controller_spec.rb+++ b/spec/requests/tags_controller_spec.rb@@ -592,6 +592,19 @@ RSpec.describe TagsController doexpect(response.redirect_url).to match(%r{/tag/test/#{tag.id}/l/top\.json\?period=daily})end+ it "preserves raw query strings in HTML and JSON redirects" do+ synonym = Fabricate(:tag, target_tag: tag)+ query = "encoded=first%20value&duplicate=one&duplicate=two&path=%2Ffoo%2fbar"++ get "/tag/#{tag.name}?#{query}"+ expect(response).to have_http_status(:moved_permanently)+ expect(response.redirect_url).to end_with("/tag/test/#{tag.id}?#{query}")++ get "/tag/#{synonym.name}/l/top.json?#{query}"+ expect(response).to have_http_status(:moved_permanently)+ expect(response.redirect_url).to end_with("/tag/test/#{tag.id}/l/top.json?#{query}")+ end+it "is not creating infinite redirect loop when tag is a synonym of itself" dotag.update!(target_tag_id: tag.id)