A platform for community discussion. Free, open, simple.
FEATURE: Include AI summaries in public topic HTML (#44250)
Natalie Tay01591cef94cb
diff --git a/app/controllers/topics_controller.rb b/app/controllers/topics_controller.rbindex 8f89d804388..a6d8509bc8f 100644--- a/app/controllers/topics_controller.rb+++ b/app/controllers/topics_controller.rb@@ -1569,7 +1569,11 @@ class TopicsController < ApplicationControllerformat.html do@tags = @topic_view.visible_tags- if SiteSetting.content_localization_enabled && use_crawler_layout?+ if SiteSetting.content_localization_enabled &&+ (+ use_crawler_layout? ||+ Middleware::AnonymousCache::Helper.new(request.env).is_crawler?+ )helpers.localize_topic_view_content(@topic_view)end@breadcrumbs = helpers.categories_breadcrumb(@topic_view.topic) || []diff --git a/app/views/topics/show.html.erb b/app/views/topics/show.html.erbindex 18df22745b6..5c70db63e2a 100644--- a/app/views/topics/show.html.erb+++ b/app/views/topics/show.html.erb@@ -54,6 +54,8 @@</div><% end %>+ <%= build_plugin_html 'server:topic-show-before-posts', topic_view: @topic_view %>+<% if main_entity_schema %><div <%= main_entity_schema %>><%= build_plugin_html 'server:topic-main-entity-meta-crawler' %>diff --git a/db/structure.sql b/db/structure.sqlindex 7cf750b9dcb..37c370bc09b 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1008,7 +1008,8 @@ CREATE TABLE public.ai_summaries (summary_type integer DEFAULT 0 NOT NULL,origin integer,highest_target_number integer DEFAULT 1 NOT NULL,- locale character varying(20)+ locale character varying(20),+ summarized_cooked text);@@ -20608,6 +20609,13 @@ CREATE UNIQUE INDEX index_ai_secrets_on_name ON public.ai_secrets USING btree (nCREATE INDEX index_ai_spam_logs_on_post_id ON public.ai_spam_logs USING btree (post_id);+--+-- Name: index_ai_summaries_missing_cooked; Type: INDEX; Schema: public; Owner: -+--++CREATE INDEX index_ai_summaries_missing_cooked ON public.ai_summaries USING btree (id) WHERE ((summarized_cooked IS NULL) AND (summary_type = 0) AND ((target_type)::text = 'Topic'::text));++---- Name: index_ai_summaries_on_target_type_and_target_id; Type: INDEX; Schema: public; Owner: ---@@ -25560,6 +25568,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20261005091527'),+('20261005091447'),('20261001073226'),('20260930114733'),('20260928103925'),diff --git a/plugins/discourse-ai/app/jobs/scheduled/cook_missing_ai_summaries.rb b/plugins/discourse-ai/app/jobs/scheduled/cook_missing_ai_summaries.rbnew file mode 100644index 00000000000..d79320b9a61--- /dev/null+++ b/plugins/discourse-ai/app/jobs/scheduled/cook_missing_ai_summaries.rb@@ -0,0 +1,27 @@+# frozen_string_literal: true++module Jobs+ class CookMissingAiSummaries < ::Jobs::Scheduled+ BATCH_SIZE = 100++ every 1.hour+ cluster_concurrency 1++ def execute(_args)+ return if !SiteSetting.discourse_ai_enabled++ cursor_key = "cook_missing_ai_summaries:last_id"+ missing = AiSummary.complete.where(target_type: "Topic", summarized_cooked: nil).order(:id)+ summaries =+ missing.where("id > ?", Discourse.redis.get(cursor_key).to_i).limit(BATCH_SIZE).to_a+ summaries = missing.limit(BATCH_SIZE).to_a if summaries.empty?++ summaries.each do |summary|+ summary.cook_missing!+ rescue => error+ Discourse.warn_exception(error, message: "Failed to cook AI summary #{summary.id}")+ end+ Discourse.redis.set(cursor_key, summaries.last.id) if summaries.present?+ end+ end+enddiff --git a/plugins/discourse-ai/app/models/ai_summary.rb b/plugins/discourse-ai/app/models/ai_summary.rbindex 52fb34d6fac..629573b9e8b 100644--- a/plugins/discourse-ai/app/models/ai_summary.rb+++ b/plugins/discourse-ai/app/models/ai_summary.rb@@ -3,6 +3,8 @@class AiSummary < ActiveRecord::Basebelongs_to :target, polymorphic: true+ before_save :refresh_cooked, if: :needs_cooking?+enum :summary_type, { complete: 0, gist: 1 }enum :origin, { human: 0, system: 1 }@@ -20,6 +22,7 @@ class AiSummary < ActiveRecord::Baseorigin: human ? origins[:human] : origins[:system],locale: strategy.locale,}+ attributes.merge!(new(attributes).cooked_attributes)strategy.target.with_lock dostored_summary = upsert_summary!(attributes)@@ -50,6 +53,7 @@ class AiSummary < ActiveRecord::Baseunique_by: %i[target_id target_type summary_type locale],update_only: %i[summarized_text+ summarized_cookedoriginal_content_shaalgorithmorigin@@ -94,6 +98,48 @@ class AiSummary < ActiveRecord::Basedef outdated@outdated || falseend++ def cooked_attributes+ return { summarized_cooked: nil } if !complete? || target_type != "Topic"++ cooking_locale = LocaleNormalizer.normalize_to_i18n(locale)+ cooking_locale = SiteSetting.default_locale if !I18n.locale_available?(cooking_locale)++ I18n.with_locale(cooking_locale) do+ { summarized_cooked: PrettyText.cook(summarized_text, topic_id: target_id) }+ end+ end++ def cook_missing!+ return if !complete? || target_type != "Topic" || !summarized_cooked.nil?++ self+ .class+ .where(+ id:,+ summarized_text:,+ updated_at:,+ locale:,+ target_type:,+ target_id:,+ summary_type:,+ summarized_cooked: nil,+ )+ .update_all(cooked_attributes)+ end++ private++ def needs_cooking?+ return summarized_cooked.present? if !complete? || target_type != "Topic"++ summarized_text_changed? || locale_changed? || target_type_changed? || target_id_changed? ||+ summarized_cooked.nil?+ end++ def refresh_cooked+ assign_attributes(cooked_attributes)+ endend# == Schema Information@@ -106,6 +152,7 @@ end# locale :string(20)# origin :integer# original_content_sha :string not null+# summarized_cooked :text# summarized_text :string not null# summary_type :integer default("complete"), not null# target_type :string not null@@ -116,5 +163,6 @@ end# Indexes## idx_ai_summaries_on_target_type_and_locale (target_id,target_type,summary_type,locale) UNIQUE NULLS NOT DISTINCT+# index_ai_summaries_missing_cooked (id) WHERE ((summarized_cooked IS NULL) AND (summary_type = 0) AND ((target_type)::text = 'Topic'::text))# index_ai_summaries_on_target_type_and_target_id (target_type,target_id)#diff --git a/plugins/discourse-ai/app/views/discourse_ai/summarization/_published_summary.html.erb b/plugins/discourse-ai/app/views/discourse_ai/summarization/_published_summary.html.erbnew file mode 100644index 00000000000..b6e08cc3d40--- /dev/null+++ b/plugins/discourse-ai/app/views/discourse_ai/summarization/_published_summary.html.erb@@ -0,0 +1,6 @@+<section class="ai-published-summary">+ <h2><%= t("discourse_ai.summarization.published_summary_heading") %></h2>+ <div class="ai-published-summary__text" itemprop="abstract">+ <%= publication.cooked.html_safe %>+ </div>+</section>diff --git a/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js b/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.jsindex 58fb9e15c82..a4607e4b93c 100644--- a/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js+++ b/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js@@ -143,6 +143,7 @@ export const AI_FEATURE_SETTING_GROUPS = {settings: ["ai_summarization_enabled","ai_summarization_agent",+ "ai_summaries_for_crawlers","ai_pm_summarization_allowed_groups",],},diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex b1ddcdaef9e..2a60b2ba699 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -147,6 +147,7 @@ en:ai_admin_dashboard_highlights_category_scope: "Choose which categories AI highlights can use. Include and exclude options use {{setting:ai_admin_dashboard_highlights_categories}}. Personal messages are always excluded."ai_admin_dashboard_highlights_categories: "Categories used when the AI highlights category scope is set to <strong>include</strong> or <strong>exclude</strong>."ai_summarization_enabled: "Enable the summarize feature"+ ai_summaries_for_crawlers: "Include AI topic summaries in public HTML topic pages served to search engines and other crawlers. Requires {{setting:ai_summarization_enabled}}."ai_summarization_agent: "Agent to use for summarize feature"ai_pm_summarization_allowed_groups: "Groups allowed to create and view summaries in PMs."ai_summary_gists_enabled: "Generate brief summaries of latest replies in topics automatically"@@ -1229,6 +1230,7 @@ en:execution_error_unknown: "an unexpected error occurred"summarization:+ published_summary_heading: "AI-generated summary"configuration_hint:one: "Configure the `%{setting}` setting first."other: "Configure these settings first: %{settings}"diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.ymlindex 586f0e24e0a..499f4eae48a 100644--- a/plugins/discourse-ai/config/settings.yml+++ b/plugins/discourse-ai/config/settings.yml@@ -347,6 +347,9 @@ discourse_ai:type: enumenum: "DiscourseAi::Configuration::AgentEnumerator"area: "ai-features/summarization"+ ai_summaries_for_crawlers:+ default: false+ area: "ai-features/summarization"ai_pm_summarization_allowed_groups:type: group_listlist_type: compactdiff --git a/plugins/discourse-ai/db/migrate/20261005091447_add_cooked_to_ai_summaries.rb b/plugins/discourse-ai/db/migrate/20261005091447_add_cooked_to_ai_summaries.rbnew file mode 100644index 00000000000..2456e51855a--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20261005091447_add_cooked_to_ai_summaries.rb@@ -0,0 +1,6 @@+# frozen_string_literal: true+class AddCookedToAiSummaries < ActiveRecord::Migration[8.1]+ def change+ add_column :ai_summaries, :summarized_cooked, :text+ end+enddiff --git a/plugins/discourse-ai/db/migrate/20261005091527_index_ai_summaries_missing_cooked.rb b/plugins/discourse-ai/db/migrate/20261005091527_index_ai_summaries_missing_cooked.rbnew file mode 100644index 00000000000..391c5c32e4d--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20261005091527_index_ai_summaries_missing_cooked.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true+class IndexAiSummariesMissingCooked < ActiveRecord::Migration[8.1]+ disable_ddl_transaction!++ def up+ remove_index :ai_summaries,+ name: "index_ai_summaries_missing_cooked",+ if_exists: true,+ algorithm: :concurrently+ add_index :ai_summaries,+ :id,+ name: "index_ai_summaries_missing_cooked",+ where: "summarized_cooked IS NULL AND summary_type = 0 AND target_type = 'Topic'",+ algorithm: :concurrently+ end++ def down+ remove_index :ai_summaries, name: "index_ai_summaries_missing_cooked", algorithm: :concurrently+ end+enddiff --git a/plugins/discourse-ai/lib/summarization/entry_point.rb b/plugins/discourse-ai/lib/summarization/entry_point.rbindex 0976ad15445..7d84072d8a1 100644--- a/plugins/discourse-ai/lib/summarization/entry_point.rb+++ b/plugins/discourse-ai/lib/summarization/entry_point.rb@@ -4,6 +4,22 @@ module DiscourseAimodule Summarizationclass EntryPointdef inject_into(plugin)+ plugin.register_html_builder("server:topic-show-before-posts") do |controller, topic_view:|+ publication =+ DiscourseAi::Summarization::PublishedSummary.new(+ topic_view,+ guardian: controller.guardian,+ )+ next if !publication.summary++ controller.render_to_string(+ partial: "discourse_ai/summarization/published_summary",+ locals: {+ publication:,+ },+ )+ end+plugin.add_to_serializer(:current_user, :can_request_gists) { scope.can_request_gists? }plugin.add_to_serializer(:current_user, :can_summarize) dodiff --git a/plugins/discourse-ai/lib/summarization/published_summary.rb b/plugins/discourse-ai/lib/summarization/published_summary.rbnew file mode 100644index 00000000000..c5031669e22--- /dev/null+++ b/plugins/discourse-ai/lib/summarization/published_summary.rb@@ -0,0 +1,46 @@+# frozen_string_literal: true++module DiscourseAi+ module Summarization+ class PublishedSummary+ def initialize(topic_view, guardian:)+ @topic_view = topic_view+ @guardian = guardian+ end++ def eligible?+ request = @guardian.request+ return false if !request || !Middleware::AnonymousCache::Helper.new(request.env).is_crawler?++ topic = @topic_view.topic+ SiteSetting.ai_summaries_for_crawlers && SiteSetting.ai_summarization_enabled &&+ !SiteSetting.login_required && @topic_view.page == 1 &&+ !@topic_view.single_post_request? && topic.regular? && topic.visible &&+ !topic.category&.read_restricted && Guardian.new.can_see?(topic)+ end++ def summary+ return @summary if defined?(@summary)++ @summary = nil+ return unless eligible?++ cached_summary =+ DiscourseAi::TopicSummarization.for(+ @topic_view.topic,+ nil,+ scope: @guardian,+ ).cached_summary+ if cached_summary&.summarized_cooked.present? &&+ Guardian.new.can_see_summary?(@topic_view.topic, cached_summary:)+ @summary = cached_summary+ end+ @summary+ end++ def cooked+ summary.summarized_cooked+ end+ end+ end+enddiff --git a/plugins/discourse-ai/spec/jobs/scheduled/cook_missing_ai_summaries_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/cook_missing_ai_summaries_spec.rbnew file mode 100644index 00000000000..9b626bbe8ec--- /dev/null+++ b/plugins/discourse-ai/spec/jobs/scheduled/cook_missing_ai_summaries_spec.rb@@ -0,0 +1,76 @@+# frozen_string_literal: true++describe Jobs::CookMissingAiSummaries do+ before { enable_current_plugin }++ it "reserves batch capacity for complete topic summaries rather than gists" do+ gists = Fabricate.times(2, :topic_ai_gist)+ summary = Fabricate(:ai_summary, summarized_text: "**Complete summary**")+ AiSummary.where(id: gists.map(&:id) + [summary.id]).update_all(summarized_cooked: nil)++ stub_const(described_class, :BATCH_SIZE, 2) { described_class.new.execute({}) }++ expect(summary.reload.summarized_cooked).to include("<strong>Complete summary</strong>")+ expect(gists.map { |gist| gist.reload.summarized_cooked }).to eq([nil, nil])+ end++ it "cooks a bounded batch of missing HTML without LLM calls or timestamp changes" do+ summaries = Fabricate.times(3, :ai_summary, summarized_text: "**Stored summary**")+ AiSummary.where(id: summaries.map(&:id)).update_all(summarized_cooked: nil)+ timestamps = summaries.map(&:updated_at)+ SiteSetting.ai_summary_backfill_maximum_topics_per_hour = 0++ stub_const(described_class, :BATCH_SIZE, 2) do+ DiscourseAi::Completions::Llm.with_prepared_responses([]) { described_class.new.execute({}) }+ end++ expect(summaries.map { |summary| summary.reload.summarized_cooked.present? }).to eq(+ [true, true, false],+ )+ expect(summaries.first.summarized_cooked).to include("<strong>Stored summary</strong>")+ expect(summaries.map(&:updated_at)).to eq(timestamps)++ described_class.new.execute({})+ expect(summaries.last.reload.summarized_cooked).to include("<strong>Stored summary</strong>")+ end++ it "does not change existing cooked HTML" do+ summary = Fabricate(:ai_summary)+ summary.update_columns(summarized_cooked: "<p>Previously cooked</p>")++ described_class.new.execute({})++ expect(summary.reload.summarized_cooked).to eq("<p>Previously cooked</p>")+ end++ it "continues past a failing batch and retries missing records on a later pass" do+ summaries = Fabricate.times(3, :ai_summary, summarized_text: "**Stored summary**")+ AiSummary.where(id: summaries.map(&:id)).update_all(summarized_cooked: nil)+ PrettyText.stubs(:cook).raises(StandardError, "Renderer unavailable")++ stub_const(described_class, :BATCH_SIZE, 2) do+ described_class.new.execute({})+ PrettyText.unstub(:cook)+ described_class.new.execute({})++ expect(summaries.map { |summary| summary.reload.summarized_cooked.present? }).to eq(+ [false, false, true],+ )++ described_class.new.execute({})+ expect(summaries.map { |summary| summary.reload.summarized_cooked.present? }).to eq(+ [true, true, true],+ )+ end+ end++ it "leaves records alone when the plugin is disabled" do+ summary = Fabricate(:ai_summary)+ summary.update_columns(summarized_cooked: nil)+ SiteSetting.discourse_ai_enabled = false++ described_class.new.execute({})++ expect(summary.reload.summarized_cooked).to be_nil+ end+enddiff --git a/plugins/discourse-ai/spec/models/ai_summary_spec.rb b/plugins/discourse-ai/spec/models/ai_summary_spec.rbindex 92e110cfc63..aee683600b0 100644--- a/plugins/discourse-ai/spec/models/ai_summary_spec.rb+++ b/plugins/discourse-ai/spec/models/ai_summary_spec.rb@@ -8,6 +8,120 @@ RSpec.describe AiSummary dobefore { enable_current_plugin }+ it "persists unsupported content languages using a supported cooking locale" do+ SiteSetting.ai_summaries_for_crawlers = false+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic, locale: "hi")+ summary =+ described_class.store!(strategy, llm_model, "**सारांश**", strategy.targets_data, human: true)++ expect(summary.locale).to eq("hi")+ expect(summary.summarized_cooked).to include("<strong>सारांश</strong>")++ summary.update!(summarized_text: "Updated **सारांश**")+ expect(summary.reload.summarized_cooked).to include("Updated <strong>सारांश</strong>")++ summary.update_columns(summarized_cooked: nil)+ timestamp = summary.updated_at+ summary.cook_missing!+ expect(summary.reload.summarized_cooked).to include("Updated <strong>सारांश</strong>")+ expect(summary.locale).to eq("hi")+ expect(summary.updated_at).to eq_time(timestamp)+ end++ it "stores and updates gist text without cooking HTML" do+ strategy = DiscourseAi::Summarization::Strategies::HotTopicGists.new(topic, locale: "en")+ summary =+ described_class.store!(strategy, llm_model, "**Gist**", strategy.targets_data, human: true)+ expect(summary.summarized_cooked).to be_nil++ summary =+ described_class.store!(+ strategy,+ llm_model,+ "**Revised gist**",+ strategy.targets_data,+ human: true,+ )+ expect(summary.summarized_text).to eq("**Revised gist**")+ expect(summary.summarized_cooked).to be_nil++ summary.update!(summarized_text: "**Edited gist**")+ summary.cook_missing!+ expect(summary.reload.summarized_cooked).to be_nil++ summary.update!(summary_type: :complete)+ expect(summary.reload.summarized_cooked).to include("<strong>Edited gist</strong>")+ summary.update!(summary_type: :gist)+ expect(summary.reload.summarized_cooked).to be_nil+ end++ it "stores sanitized cooked content and replaces it when regenerating a summary" do+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic, locale: "en")+ summary =+ described_class.store!(+ strategy,+ llm_model,+ "**Overview** <script>alert(1)</script>",+ strategy.targets_data,+ human: true,+ )++ expect(summary.summarized_cooked).to include("<strong>Overview</strong>")+ expect(summary.summarized_cooked).not_to include("<script>")++ regenerated =+ described_class.store!(+ strategy,+ llm_model,+ "A revised **summary**",+ strategy.targets_data,+ human: true,+ )++ expect(regenerated.id).to eq(summary.id)+ expect(regenerated.summarized_cooked).to include("A revised <strong>summary</strong>")+ expect(regenerated.summarized_cooked).not_to include("Overview")+ end++ it "recooks changed text on ordinary saves" do+ summary = Fabricate(:ai_summary, target: topic)+ summary.update!(summarized_text: "Updated **text**")++ expect(summary.reload.summarized_cooked).to include("Updated <strong>text</strong>")+ end++ it "cooks missing HTML without changing the summary freshness timestamp" do+ summary = Fabricate(:ai_summary, target: topic, summarized_text: "**Original**")+ summary.update_columns(summarized_cooked: nil)+ updated_at = summary.updated_at++ summary.cook_missing!++ expect(summary.reload.summarized_cooked).to include("<strong>Original</strong>")+ expect(summary.updated_at).to eq_time(updated_at)+ end++ it "does not overwrite a regenerated summary when cooking an older snapshot" do+ summary = Fabricate(:ai_summary, target: topic, summarized_text: "Old summary")+ summary.update_columns(summarized_cooked: nil)+ snapshot = described_class.find(summary.id)+ summary.update!(summarized_text: "New **summary**")++ snapshot.cook_missing!++ expect(summary.reload.summarized_cooked).to include("New <strong>summary</strong>")+ expect(summary.summarized_text).to eq("New **summary**")+ end++ it "leaves existing cooked HTML alone" do+ summary = Fabricate(:ai_summary, target: topic)+ summary.update_columns(summarized_cooked: "<p>Previously cooked</p>")++ summary.cook_missing!++ expect(summary.reload.summarized_cooked).to eq("<p>Previously cooked</p>")+ end+it "stores and independently upserts topic gists by locale" doenglish_strategy =DiscourseAi::Summarization::Strategies::HotTopicGists.new(topic, locale: "en")diff --git a/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb b/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rbnew file mode 100644index 00000000000..6a05a5709c5--- /dev/null+++ b/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb@@ -0,0 +1,475 @@+# frozen_string_literal: true++describe TopicsController do+ fab!(:topic)+ fab!(:first_post) { Fabricate(:post, topic: topic, post_number: 1) }++ let(:summary_phrase) { "concise overview" }+ let(:summary_text) { "A **#{summary_phrase}** of this discussion." }+ let(:crawler_headers) { { "User-Agent" => "Googlebot" } }+ let(:browser_headers) { { "User-Agent" => "Mozilla/5.0 Chrome/130.0.0.0 Safari/537.36" } }++ before do+ enable_current_plugin+ assign_fake_provider_to(:ai_default_llm_model)+ assign_agent_to(:ai_summarization_agent, [Group::AUTO_GROUPS[:admins]])+ SiteSetting.ai_summarization_enabled = true+ SiteSetting.enable_markdown_endpoints = true+ SiteSetting.ai_summaries_for_crawlers = true+ end++ def create_summary(locale: "en", text: summary_text)+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic, locale:)+ Fabricate(+ :ai_summary,+ target: topic,+ locale:,+ summarized_text: text,+ original_content_sha: strategy.summary_fingerprint[:original_content_sha],+ highest_target_number: topic.highest_post_number,+ )+ end++ describe "#show" do+ it "keeps forced crawler translations consistent on misses and shared cache hits" do+ global_setting :anon_cache_store_threshold, 1+ Middleware::AnonymousCache.enable_anon_cache+ SiteSetting.content_localization_enabled = true+ SiteSetting.content_localization_supported_locales = "fr"+ SiteSetting.default_locale = "fr"+ topic.update!(locale: "en")+ first_post.update!(locale: "en")+ Fabricate(:post_localization, post: first_post, locale: "fr", cooked: "<p>Bonjour</p>")+ create_summary+ french = create_summary(locale: "fr", text: "Un résumé de cette discussion.")+ forced_headers = browser_headers.merge("Discourse-Render" => "crawler")+ bot_headers = { "User-Agent" => "#{browser_headers["User-Agent"]} Googlebot" }++ %w[true false].each do |translate|+ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = translate+ [[forced_headers, bot_headers], [bot_headers, forced_headers]].each do |headers|+ Middleware::AnonymousCache.clear_all_cache!+ headers.each_with_index do |request_headers, index|+ get topic.relative_url, headers: request_headers++ expect(response.status).to eq(200)+ expect(response.headers["X-Discourse-Cached"]).to eq(index.zero? ? "store" : "true")+ document = Nokogiri.HTML5(response.body)+ expect(document.at_css('[itemprop="abstract"]').text).to include(+ translate == "true" ? french.summarized_text : summary_phrase,+ )+ expect(document.at_css("#post_1 .post").text).to include(+ translate == "true" ? "Bonjour" : first_post.raw,+ )+ end+ end+ end+ end++ it "publishes a cached summary as the discussion abstract in crawler HTML" do+ create_summary++ get topic.relative_url, headers: crawler_headers++ expect(response.status).to eq(200)+ document = Nokogiri.HTML5(response.body)+ abstract =+ document.at_css(+ '[itemtype="http://schema.org/DiscussionForumPosting"] > section [itemprop="abstract"]',+ )+ expect(abstract&.text).to include("A #{summary_phrase} of this discussion.")+ expect(abstract.at_css("strong").text).to eq(summary_phrase)+ expect(document.at_css("#post_1 .post").text).to include(first_post.raw)+ end++ it "leaves crawler Markdown responses unchanged" do+ create_summary++ get "#{topic.relative_url}.md", headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).not_to include("## AI-generated summary", summary_phrase)+ expect(response.body).to include(first_post.raw)+ end++ it "uses the same crawler classification as the anonymous cache" do+ global_setting :anon_cache_store_threshold, 1+ Middleware::AnonymousCache.enable_anon_cache+ Middleware::AnonymousCache.clear_all_cache!+ create_summary++ get topic.relative_url, headers: browser_headers.merge("Discourse-Render" => "crawler")++ expect(response.headers["X-Discourse-Cached"]).to eq("store")+ expect(response.body).to include(summary_phrase)++ get topic.relative_url,+ headers: {+ "User-Agent" => "#{browser_headers["User-Agent"]} Googlebot",+ }++ expect(response.headers["X-Discourse-Cached"]).to eq("true")+ expect(response.body).to include(summary_phrase)+ end++ context "with HTML publication" do+ def request_topic(path: topic.relative_url, params: {})+ get path, params:, headers: crawler_headers+ end++ it "serves stored cooked content rather than cooking the source on request" do+ summary = create_summary+ stored_content = "Previously cooked overview"+ summary.update_columns(summarized_cooked: "<p>#{stored_content}</p>")++ request_topic++ expect(response.status).to eq(200)+ expect(response.body).to include(stored_content)+ expect(response.body).not_to include(summary_phrase)+ end++ it "omits uncooked summaries without cooking them on request" do+ summary = create_summary+ summary.update_columns(summarized_cooked: nil)++ request_topic++ expect(response.status).to eq(200)+ expect(response.body).to include(first_post.raw)+ expect(response.body).not_to include(summary_phrase)+ expect(summary.reload.summarized_cooked).to be_nil++ summary.update_columns(summarized_cooked: "<p>#{summary_phrase}</p>")+ request_topic++ expect(response.body).to include(summary_phrase)+ end++ it "leaves browser and print responses unchanged" do+ create_summary+ path = topic.relative_url++ [{}, { print: true }].each do |params|+ SiteSetting.ai_summaries_for_crawlers = false+ get path, params: params, headers: browser_headers+ original_last_modified = response.headers["Last-Modified"]+ SiteSetting.ai_summaries_for_crawlers = true++ get path, params: params, headers: browser_headers++ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)+ expect(response.headers["Last-Modified"]).to eq(original_last_modified)+ end+ end++ it "adds one summary lookup per request without retaining it across requests" do+ create_summary+ SiteSetting.ai_summaries_for_crawlers = false+ baseline_queries = track_sql_queries { request_topic }+ SiteSetting.ai_summaries_for_crawlers = true++ queries = track_sql_queries { request_topic }++ expect(response.status).to eq(200)+ expect(response.body).to include(summary_phrase)+ summary_query = /SELECT.*FROM "ai_summaries"/+ expect(queries.grep(summary_query).size).to eq(+ baseline_queries.grep(summary_query).size + 1,+ )++ AiSummary.where(target: topic).delete_all+ request_topic++ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)++ create_summary+ request_topic++ expect(response.status).to eq(200)+ expect(response.body).to include(summary_phrase)+ end++ it "serves summaries through the existing anonymous topic cache" do+ global_setting :anon_cache_store_threshold, 1+ Middleware::AnonymousCache.enable_anon_cache+ Middleware::AnonymousCache.clear_all_cache!+ summary = create_summary++ request_topic++ expect(response.headers["X-Discourse-Cached"]).to eq("store")+ expect(request.env["ANON_CACHE_DURATION"]).to eq(1.minute)+ expect(response.body).to include(summary_phrase)++ path = topic.relative_url+ get path, headers: browser_headers+ expect(response.headers["X-Discourse-Cached"]).to eq("store")+ expect(response.body).not_to include(summary_phrase)++ get path, headers: browser_headers+ expect(response.headers["X-Discourse-Cached"]).to eq("true")+ expect(response.body).not_to include(summary_phrase)++ first_post.update!(last_version_at: summary.updated_at + 1.minute)+ queries = track_sql_queries { request_topic }++ expect(response.headers["X-Discourse-Cached"]).to eq("true")+ expect(response.body).to include(summary_phrase)+ expect(queries.grep(/\b(?:ai_summaries|posts|topics)\b/)).to be_empty++ Middleware::AnonymousCache.clear_all_cache!+ request_topic++ expect(response.headers["X-Discourse-Cached"]).to eq("store")+ expect(response.body).not_to include(summary_phrase)++ request_topic++ expect(response.headers["X-Discourse-Cached"]).to eq("true")+ expect(response.body).not_to include(summary_phrase)+ end++ it "omits summaries when publication, summarization, or the plugin is disabled" do+ summary = create_summary++ %i[+ ai_summaries_for_crawlers+ ai_summarization_enabled+ discourse_ai_enabled+ ].each do |setting|+ SiteSetting.public_send("#{setting}=", false)+ request_topic++ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary.summarized_text, summary_phrase)+ SiteSetting.public_send("#{setting}=", true)+ end+ end++ it "leaves a missing summary absent without generating one" do+ DiscourseAi::Completions::Llm.with_prepared_responses([]) do+ expect { request_topic }.not_to change(AiSummary, :count)+ end++ expect(response.status).to eq(200)+ expect(response.body).to include(first_post.raw)+ expect(response.body).not_to include(+ I18n.t("discourse_ai.summarization.published_summary_heading"),+ )+ expect(response.headers["Last-Modified"]).to eq(first_post.updated_at.httpdate)+ expect(request.env["ANON_CACHE_DURATION"]).to eq(1.minute)+ end++ it "omits an edited summary even for an admin and preserves the post freshness hint" do+ summary = create_summary+ sign_in(Fabricate(:admin))+ request_topic+ expect(response.body).to include(summary_phrase)++ first_post.update!(last_version_at: summary.updated_at + 1.minute)+ request_topic++ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)+ expect(response.headers["Last-Modified"]).to eq(first_post.reload.updated_at.httpdate)+ end++ it "omits a summary after new replies or hidden source posts change the input" do+ summary = create_summary+ reply = Fabricate(:post, topic:, post_number: 2)+ topic.update!(highest_post_number: 2)+ request_topic+ expect(response.body).not_to include(summary_phrase)++ summary.destroy!+ create_summary+ request_topic+ expect(response.body).to include(summary_phrase)++ reply.update!(hidden: true)+ request_topic+ expect(response.body).not_to include(summary_phrase)+ end++ it "omits a summary after a source post is deleted" do+ reply = Fabricate(:post, topic:, post_number: 2)+ topic.update!(highest_post_number: 2)+ create_summary+ request_topic+ expect(response.body).to include(summary_phrase)++ reply.trash!+ request_topic++ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)+ expect(response.headers["Last-Modified"]).to eq(first_post.reload.updated_at.httpdate)+ end++ it "omits restricted topics even when the requester is an admin" do+ create_summary+ topic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))+ sign_in(Fabricate(:admin))++ request_topic++ expect(response.status).to eq(200)+ expect(response.body).to include(first_post.raw)+ expect(response.body).not_to include(summary_phrase)+ end++ it "omits summaries when the site requires login" do+ create_summary+ SiteSetting.login_required = true+ sign_in(Fabricate(:admin))++ request_topic++ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)+ end++ it "omits personal messages even when their participant can summarize" do+ admin = Fabricate(:admin)+ message = Fabricate(:private_message_topic, user: admin)+ message_post = Fabricate(:post, topic: message)+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(message, locale: "en")+ Fabricate(+ :ai_summary,+ target: message,+ locale: "en",+ summarized_text: summary_text,+ original_content_sha: strategy.summary_fingerprint[:original_content_sha],+ )+ SiteSetting.ai_pm_summarization_allowed_groups = Group::AUTO_GROUPS[:admins].to_s+ sign_in(admin)++ request_topic(path: message.relative_url)++ expect(response.status).to eq(200)+ expect(response.body).to include(message_post.raw)+ expect(response.body).not_to include(summary_phrase)+ end++ it "omits summaries from later pages and single-post responses" do+ Fabricate.times(TopicView.chunk_size, :post, topic:)+ topic.update!(highest_post_number: TopicView.chunk_size + 1)+ create_summary++ request_topic(params: { page: 2 })+ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)++ request_topic(path: "#{topic.relative_url}/2")+ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)+ end++ it "leaves Last-Modified and topic activity unchanged when a summary is regenerated" do+ freeze_time+ topic.update!(last_posted_at: first_post.created_at)+ summary = create_summary+ request_topic+ previous_last_modified = response.headers["Last-Modified"]+ topic_activity = topic.reload.last_posted_at++ freeze_time 2.minutes.from_now+ summary.update!(summarized_text: "An updated overview")+ request_topic++ expect(response.body).to include(summary.summarized_text)+ expect(response.headers["Last-Modified"]).to eq(previous_last_modified)+ expect(topic.reload.last_posted_at).to eq_time(topic_activity)+ expect(request.env["ANON_CACHE_DURATION"]).to eq(1.minute)+ end++ it "uses the source-language summary instead of another cached locale" do+ topic.update!(locale: "fr")+ english = create_summary+ french = create_summary(locale: "fr", text: "Un résumé de cette discussion.")++ request_topic++ expect(response.status).to eq(200)+ expect(response.body).to include(french.summarized_text)+ expect(response.body).not_to include(english.summarized_text, summary_phrase)+ end++ it "uses a localized summary when the discussion is shown translated" do+ SiteSetting.content_localization_enabled = true+ SiteSetting.content_localization_supported_locales = "fr"+ SiteSetting.default_locale = "fr"+ topic.update!(locale: "en")+ first_post.update!(locale: "en")+ Fabricate(:post_localization, post: first_post, locale: "fr", cooked: "<p>Bonjour</p>")+ english = create_summary+ french = create_summary(locale: "fr", text: "Un résumé de cette discussion.")++ request_topic++ expect(response.status).to eq(200)+ expect(response.body).to include(french.summarized_text)+ expect(response.body).not_to include(english.summarized_text, summary_phrase)++ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = "false"+ request_topic++ expect(response.body).to include(summary_phrase, first_post.raw)+ expect(response.body).not_to include(french.summarized_text, "Bonjour")++ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = "true"+ french.destroy!+ request_topic+ expect(response.body).not_to include(+ english.summarized_text,+ french.summarized_text,+ summary_phrase,+ )+ end+ end++ it "sanitizes summary markup in HTML" do+ create_summary(+ text: 'Overview <script>alert("xss")</script> <a href="javascript:alert(1)">unsafe</a>',+ )++ get topic.relative_url, headers: crawler_headers+ abstract = Nokogiri.HTML5(response.body).at_css('[itemprop="abstract"]')+ expect(abstract.text).to include("Overview")+ expect(abstract.css("script, [href^='javascript:']")).to be_empty+ end++ it "leaves topic JSON unchanged" do+ summary = create_summary+ summary.update!(updated_at: first_post.updated_at + 1.minute)++ get "#{topic.relative_url}.json", headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.parsed_body["has_cached_summary"]).to eq(true)+ expect(response.parsed_body).not_to have_key("ai_summary")+ expect(response.headers["Last-Modified"]).to be_nil+ end++ it "keeps the abstract on QAPage rather than the nested question or answers" do+ SiteSetting.allow_solved_on_all_topics = true+ SiteSetting.solved_add_schema_markup = "always"+ Fabricate(:post, topic:, post_number: 2)+ topic.update!(highest_post_number: 2)+ create_summary++ get topic.relative_url, headers: crawler_headers++ expect(response.status).to eq(200)+ document = Nokogiri.HTML5(response.body)+ abstract = document.at_css('[itemprop="abstract"]')+ expect(abstract.ancestors("[itemscope]").first["itemtype"]).to eq("https://schema.org/QAPage")+ expect(+ document.css('[itemtype="https://schema.org/Question"] [itemprop="abstract"]'),+ ).to be_empty+ end+ end+enddiff --git a/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.js b/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.jsindex 4ce7d184809..a1e94a2c3e0 100644--- a/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.js+++ b/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.js@@ -66,6 +66,10 @@ module("Unit | Lib | ai-feature-setting-groups", function () {const groups = getSettingGroupsForFeature("summarization");assert.strictEqual(groups.length, 3, "summarization should have 3 groups");+ assert.true(+ groups[0].settings.includes("ai_summaries_for_crawlers"),+ "basic settings should include summary publication"+ );});test("returns correct groups for search", function (assert) {diff --git a/spec/requests/topics_controller_spec.rb b/spec/requests/topics_controller_spec.rbindex ff9c7c0f4c1..efbaa14d8d4 100644--- a/spec/requests/topics_controller_spec.rb+++ b/spec/requests/topics_controller_spec.rb@@ -7943,6 +7943,29 @@ RSpec.describe TopicsController doexpect(response.body).to include(subcategory.name)expect(response.body).to include(topic.first_post.cooked)end++ it "localizes forced crawler HTML while respecting translation opt-out" do+ SiteSetting.default_locale = "pt"+ headers = {+ "User-Agent" => "Mozilla/5.0 Chrome/130.0.0.0 Safari/537.36",+ "Discourse-Render" => "crawler",+ }++ get topic.relative_url, headers: headers++ document = Nokogiri.HTML5(response.body)+ expect(document.at_css("#post_1 .post").text).to include(+ Nokogiri::HTML5.fragment(pt_first_post.cooked).text,+ )++ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = "false"+ get topic.relative_url, headers: headers++ document = Nokogiri.HTML5(response.body)+ expect(document.at_css("#post_1 .post").text).to include(+ Nokogiri::HTML5.fragment(topic.first_post.cooked).text,+ )+ endenddescribe "when tl param is present ?tl=ja" do