Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: Include AI summaries in public topic HTML (#44250)

Natalie Tay01591cef94cb
diff --git a/app/controllers/topics_controller.rb b/app/controllers/topics_controller.rb
index 8f89d804388..a6d8509bc8f 100644
--- a/app/controllers/topics_controller.rb
+++ b/app/controllers/topics_controller.rb
@@ -1569,7 +1569,11 @@ class TopicsController < ApplicationController
format.html do
@tags = @topic_view.visible_tags
- if SiteSetting.content_localization_enabled && use_crawler_layout?
+ if SiteSetting.content_localization_enabled &&
+ (
+ use_crawler_layout? ||
+ Middleware::AnonymousCache::Helper.new(request.env).is_crawler?
+ )
helpers.localize_topic_view_content(@topic_view)
end
@breadcrumbs = helpers.categories_breadcrumb(@topic_view.topic) || []
diff --git a/app/views/topics/show.html.erb b/app/views/topics/show.html.erb
index 18df22745b6..5c70db63e2a 100644
--- a/app/views/topics/show.html.erb
+++ b/app/views/topics/show.html.erb
@@ -54,6 +54,8 @@
</div>
<% end %>
+ <%= build_plugin_html 'server:topic-show-before-posts', topic_view: @topic_view %>
+
<% if main_entity_schema %>
<div <%= main_entity_schema %>>
<%= build_plugin_html 'server:topic-main-entity-meta-crawler' %>
diff --git a/db/structure.sql b/db/structure.sql
index 7cf750b9dcb..37c370bc09b 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1008,7 +1008,8 @@ CREATE TABLE public.ai_summaries (
summary_type integer DEFAULT 0 NOT NULL,
origin integer,
highest_target_number integer DEFAULT 1 NOT NULL,
- locale character varying(20)
+ locale character varying(20),
+ summarized_cooked text
);
@@ -20608,6 +20609,13 @@ CREATE UNIQUE INDEX index_ai_secrets_on_name ON public.ai_secrets USING btree (n
CREATE INDEX index_ai_spam_logs_on_post_id ON public.ai_spam_logs USING btree (post_id);
+--
+-- Name: index_ai_summaries_missing_cooked; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_ai_summaries_missing_cooked ON public.ai_summaries USING btree (id) WHERE ((summarized_cooked IS NULL) AND (summary_type = 0) AND ((target_type)::text = 'Topic'::text));
+
+
--
-- Name: index_ai_summaries_on_target_type_and_target_id; Type: INDEX; Schema: public; Owner: -
--
@@ -25560,6 +25568,8 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20261005091527'),
+('20261005091447'),
('20261001073226'),
('20260930114733'),
('20260928103925'),
diff --git a/plugins/discourse-ai/app/jobs/scheduled/cook_missing_ai_summaries.rb b/plugins/discourse-ai/app/jobs/scheduled/cook_missing_ai_summaries.rb
new file mode 100644
index 00000000000..d79320b9a61
--- /dev/null
+++ b/plugins/discourse-ai/app/jobs/scheduled/cook_missing_ai_summaries.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Jobs
+ class CookMissingAiSummaries < ::Jobs::Scheduled
+ BATCH_SIZE = 100
+
+ every 1.hour
+ cluster_concurrency 1
+
+ def execute(_args)
+ return if !SiteSetting.discourse_ai_enabled
+
+ cursor_key = "cook_missing_ai_summaries:last_id"
+ missing = AiSummary.complete.where(target_type: "Topic", summarized_cooked: nil).order(:id)
+ summaries =
+ missing.where("id > ?", Discourse.redis.get(cursor_key).to_i).limit(BATCH_SIZE).to_a
+ summaries = missing.limit(BATCH_SIZE).to_a if summaries.empty?
+
+ summaries.each do |summary|
+ summary.cook_missing!
+ rescue => error
+ Discourse.warn_exception(error, message: "Failed to cook AI summary #{summary.id}")
+ end
+ Discourse.redis.set(cursor_key, summaries.last.id) if summaries.present?
+ end
+ end
+end
diff --git a/plugins/discourse-ai/app/models/ai_summary.rb b/plugins/discourse-ai/app/models/ai_summary.rb
index 52fb34d6fac..629573b9e8b 100644
--- a/plugins/discourse-ai/app/models/ai_summary.rb
+++ b/plugins/discourse-ai/app/models/ai_summary.rb
@@ -3,6 +3,8 @@
class AiSummary < ActiveRecord::Base
belongs_to :target, polymorphic: true
+ before_save :refresh_cooked, if: :needs_cooking?
+
enum :summary_type, { complete: 0, gist: 1 }
enum :origin, { human: 0, system: 1 }
@@ -20,6 +22,7 @@ class AiSummary < ActiveRecord::Base
origin: human ? origins[:human] : origins[:system],
locale: strategy.locale,
}
+ attributes.merge!(new(attributes).cooked_attributes)
strategy.target.with_lock do
stored_summary = upsert_summary!(attributes)
@@ -50,6 +53,7 @@ class AiSummary < ActiveRecord::Base
unique_by: %i[target_id target_type summary_type locale],
update_only: %i[
summarized_text
+ summarized_cooked
original_content_sha
algorithm
origin
@@ -94,6 +98,48 @@ class AiSummary < ActiveRecord::Base
def outdated
@outdated || false
end
+
+ def cooked_attributes
+ return { summarized_cooked: nil } if !complete? || target_type != "Topic"
+
+ cooking_locale = LocaleNormalizer.normalize_to_i18n(locale)
+ cooking_locale = SiteSetting.default_locale if !I18n.locale_available?(cooking_locale)
+
+ I18n.with_locale(cooking_locale) do
+ { summarized_cooked: PrettyText.cook(summarized_text, topic_id: target_id) }
+ end
+ end
+
+ def cook_missing!
+ return if !complete? || target_type != "Topic" || !summarized_cooked.nil?
+
+ self
+ .class
+ .where(
+ id:,
+ summarized_text:,
+ updated_at:,
+ locale:,
+ target_type:,
+ target_id:,
+ summary_type:,
+ summarized_cooked: nil,
+ )
+ .update_all(cooked_attributes)
+ end
+
+ private
+
+ def needs_cooking?
+ return summarized_cooked.present? if !complete? || target_type != "Topic"
+
+ summarized_text_changed? || locale_changed? || target_type_changed? || target_id_changed? ||
+ summarized_cooked.nil?
+ end
+
+ def refresh_cooked
+ assign_attributes(cooked_attributes)
+ end
end
# == Schema Information
@@ -106,6 +152,7 @@ end
# locale :string(20)
# origin :integer
# original_content_sha :string not null
+# summarized_cooked :text
# summarized_text :string not null
# summary_type :integer default("complete"), not null
# target_type :string not null
@@ -116,5 +163,6 @@ end
# Indexes
#
# idx_ai_summaries_on_target_type_and_locale (target_id,target_type,summary_type,locale) UNIQUE NULLS NOT DISTINCT
+# index_ai_summaries_missing_cooked (id) WHERE ((summarized_cooked IS NULL) AND (summary_type = 0) AND ((target_type)::text = 'Topic'::text))
# index_ai_summaries_on_target_type_and_target_id (target_type,target_id)
#
diff --git a/plugins/discourse-ai/app/views/discourse_ai/summarization/_published_summary.html.erb b/plugins/discourse-ai/app/views/discourse_ai/summarization/_published_summary.html.erb
new file mode 100644
index 00000000000..b6e08cc3d40
--- /dev/null
+++ b/plugins/discourse-ai/app/views/discourse_ai/summarization/_published_summary.html.erb
@@ -0,0 +1,6 @@
+<section class="ai-published-summary">
+ <h2><%= t("discourse_ai.summarization.published_summary_heading") %></h2>
+ <div class="ai-published-summary__text" itemprop="abstract">
+ <%= publication.cooked.html_safe %>
+ </div>
+</section>
diff --git a/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js b/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js
index 58fb9e15c82..a4607e4b93c 100644
--- a/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js
+++ b/plugins/discourse-ai/assets/javascripts/discourse/lib/ai-feature-setting-groups.js
@@ -143,6 +143,7 @@ export const AI_FEATURE_SETTING_GROUPS = {
settings: [
"ai_summarization_enabled",
"ai_summarization_agent",
+ "ai_summaries_for_crawlers",
"ai_pm_summarization_allowed_groups",
],
},
diff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.yml
index b1ddcdaef9e..2a60b2ba699 100644
--- a/plugins/discourse-ai/config/locales/server.en.yml
+++ b/plugins/discourse-ai/config/locales/server.en.yml
@@ -147,6 +147,7 @@ en:
ai_admin_dashboard_highlights_category_scope: "Choose which categories AI highlights can use. Include and exclude options use {{setting:ai_admin_dashboard_highlights_categories}}. Personal messages are always excluded."
ai_admin_dashboard_highlights_categories: "Categories used when the AI highlights category scope is set to <strong>include</strong> or <strong>exclude</strong>."
ai_summarization_enabled: "Enable the summarize feature"
+ ai_summaries_for_crawlers: "Include AI topic summaries in public HTML topic pages served to search engines and other crawlers. Requires {{setting:ai_summarization_enabled}}."
ai_summarization_agent: "Agent to use for summarize feature"
ai_pm_summarization_allowed_groups: "Groups allowed to create and view summaries in PMs."
ai_summary_gists_enabled: "Generate brief summaries of latest replies in topics automatically"
@@ -1229,6 +1230,7 @@ en:
execution_error_unknown: "an unexpected error occurred"
summarization:
+ published_summary_heading: "AI-generated summary"
configuration_hint:
one: "Configure the `%{setting}` setting first."
other: "Configure these settings first: %{settings}"
diff --git a/plugins/discourse-ai/config/settings.yml b/plugins/discourse-ai/config/settings.yml
index 586f0e24e0a..499f4eae48a 100644
--- a/plugins/discourse-ai/config/settings.yml
+++ b/plugins/discourse-ai/config/settings.yml
@@ -347,6 +347,9 @@ discourse_ai:
type: enum
enum: "DiscourseAi::Configuration::AgentEnumerator"
area: "ai-features/summarization"
+ ai_summaries_for_crawlers:
+ default: false
+ area: "ai-features/summarization"
ai_pm_summarization_allowed_groups:
type: group_list
list_type: compact
diff --git a/plugins/discourse-ai/db/migrate/20261005091447_add_cooked_to_ai_summaries.rb b/plugins/discourse-ai/db/migrate/20261005091447_add_cooked_to_ai_summaries.rb
new file mode 100644
index 00000000000..2456e51855a
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20261005091447_add_cooked_to_ai_summaries.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+class AddCookedToAiSummaries < ActiveRecord::Migration[8.1]
+ def change
+ add_column :ai_summaries, :summarized_cooked, :text
+ end
+end
diff --git a/plugins/discourse-ai/db/migrate/20261005091527_index_ai_summaries_missing_cooked.rb b/plugins/discourse-ai/db/migrate/20261005091527_index_ai_summaries_missing_cooked.rb
new file mode 100644
index 00000000000..391c5c32e4d
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20261005091527_index_ai_summaries_missing_cooked.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+class IndexAiSummariesMissingCooked < ActiveRecord::Migration[8.1]
+ disable_ddl_transaction!
+
+ def up
+ remove_index :ai_summaries,
+ name: "index_ai_summaries_missing_cooked",
+ if_exists: true,
+ algorithm: :concurrently
+ add_index :ai_summaries,
+ :id,
+ name: "index_ai_summaries_missing_cooked",
+ where: "summarized_cooked IS NULL AND summary_type = 0 AND target_type = 'Topic'",
+ algorithm: :concurrently
+ end
+
+ def down
+ remove_index :ai_summaries, name: "index_ai_summaries_missing_cooked", algorithm: :concurrently
+ end
+end
diff --git a/plugins/discourse-ai/lib/summarization/entry_point.rb b/plugins/discourse-ai/lib/summarization/entry_point.rb
index 0976ad15445..7d84072d8a1 100644
--- a/plugins/discourse-ai/lib/summarization/entry_point.rb
+++ b/plugins/discourse-ai/lib/summarization/entry_point.rb
@@ -4,6 +4,22 @@ module DiscourseAi
module Summarization
class EntryPoint
def inject_into(plugin)
+ plugin.register_html_builder("server:topic-show-before-posts") do |controller, topic_view:|
+ publication =
+ DiscourseAi::Summarization::PublishedSummary.new(
+ topic_view,
+ guardian: controller.guardian,
+ )
+ next if !publication.summary
+
+ controller.render_to_string(
+ partial: "discourse_ai/summarization/published_summary",
+ locals: {
+ publication:,
+ },
+ )
+ end
+
plugin.add_to_serializer(:current_user, :can_request_gists) { scope.can_request_gists? }
plugin.add_to_serializer(:current_user, :can_summarize) do
diff --git a/plugins/discourse-ai/lib/summarization/published_summary.rb b/plugins/discourse-ai/lib/summarization/published_summary.rb
new file mode 100644
index 00000000000..c5031669e22
--- /dev/null
+++ b/plugins/discourse-ai/lib/summarization/published_summary.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Summarization
+ class PublishedSummary
+ def initialize(topic_view, guardian:)
+ @topic_view = topic_view
+ @guardian = guardian
+ end
+
+ def eligible?
+ request = @guardian.request
+ return false if !request || !Middleware::AnonymousCache::Helper.new(request.env).is_crawler?
+
+ topic = @topic_view.topic
+ SiteSetting.ai_summaries_for_crawlers && SiteSetting.ai_summarization_enabled &&
+ !SiteSetting.login_required && @topic_view.page == 1 &&
+ !@topic_view.single_post_request? && topic.regular? && topic.visible &&
+ !topic.category&.read_restricted && Guardian.new.can_see?(topic)
+ end
+
+ def summary
+ return @summary if defined?(@summary)
+
+ @summary = nil
+ return unless eligible?
+
+ cached_summary =
+ DiscourseAi::TopicSummarization.for(
+ @topic_view.topic,
+ nil,
+ scope: @guardian,
+ ).cached_summary
+ if cached_summary&.summarized_cooked.present? &&
+ Guardian.new.can_see_summary?(@topic_view.topic, cached_summary:)
+ @summary = cached_summary
+ end
+ @summary
+ end
+
+ def cooked
+ summary.summarized_cooked
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/spec/jobs/scheduled/cook_missing_ai_summaries_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/cook_missing_ai_summaries_spec.rb
new file mode 100644
index 00000000000..9b626bbe8ec
--- /dev/null
+++ b/plugins/discourse-ai/spec/jobs/scheduled/cook_missing_ai_summaries_spec.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+describe Jobs::CookMissingAiSummaries do
+ before { enable_current_plugin }
+
+ it "reserves batch capacity for complete topic summaries rather than gists" do
+ gists = Fabricate.times(2, :topic_ai_gist)
+ summary = Fabricate(:ai_summary, summarized_text: "**Complete summary**")
+ AiSummary.where(id: gists.map(&:id) + [summary.id]).update_all(summarized_cooked: nil)
+
+ stub_const(described_class, :BATCH_SIZE, 2) { described_class.new.execute({}) }
+
+ expect(summary.reload.summarized_cooked).to include("<strong>Complete summary</strong>")
+ expect(gists.map { |gist| gist.reload.summarized_cooked }).to eq([nil, nil])
+ end
+
+ it "cooks a bounded batch of missing HTML without LLM calls or timestamp changes" do
+ summaries = Fabricate.times(3, :ai_summary, summarized_text: "**Stored summary**")
+ AiSummary.where(id: summaries.map(&:id)).update_all(summarized_cooked: nil)
+ timestamps = summaries.map(&:updated_at)
+ SiteSetting.ai_summary_backfill_maximum_topics_per_hour = 0
+
+ stub_const(described_class, :BATCH_SIZE, 2) do
+ DiscourseAi::Completions::Llm.with_prepared_responses([]) { described_class.new.execute({}) }
+ end
+
+ expect(summaries.map { |summary| summary.reload.summarized_cooked.present? }).to eq(
+ [true, true, false],
+ )
+ expect(summaries.first.summarized_cooked).to include("<strong>Stored summary</strong>")
+ expect(summaries.map(&:updated_at)).to eq(timestamps)
+
+ described_class.new.execute({})
+ expect(summaries.last.reload.summarized_cooked).to include("<strong>Stored summary</strong>")
+ end
+
+ it "does not change existing cooked HTML" do
+ summary = Fabricate(:ai_summary)
+ summary.update_columns(summarized_cooked: "<p>Previously cooked</p>")
+
+ described_class.new.execute({})
+
+ expect(summary.reload.summarized_cooked).to eq("<p>Previously cooked</p>")
+ end
+
+ it "continues past a failing batch and retries missing records on a later pass" do
+ summaries = Fabricate.times(3, :ai_summary, summarized_text: "**Stored summary**")
+ AiSummary.where(id: summaries.map(&:id)).update_all(summarized_cooked: nil)
+ PrettyText.stubs(:cook).raises(StandardError, "Renderer unavailable")
+
+ stub_const(described_class, :BATCH_SIZE, 2) do
+ described_class.new.execute({})
+ PrettyText.unstub(:cook)
+ described_class.new.execute({})
+
+ expect(summaries.map { |summary| summary.reload.summarized_cooked.present? }).to eq(
+ [false, false, true],
+ )
+
+ described_class.new.execute({})
+ expect(summaries.map { |summary| summary.reload.summarized_cooked.present? }).to eq(
+ [true, true, true],
+ )
+ end
+ end
+
+ it "leaves records alone when the plugin is disabled" do
+ summary = Fabricate(:ai_summary)
+ summary.update_columns(summarized_cooked: nil)
+ SiteSetting.discourse_ai_enabled = false
+
+ described_class.new.execute({})
+
+ expect(summary.reload.summarized_cooked).to be_nil
+ end
+end
diff --git a/plugins/discourse-ai/spec/models/ai_summary_spec.rb b/plugins/discourse-ai/spec/models/ai_summary_spec.rb
index 92e110cfc63..aee683600b0 100644
--- a/plugins/discourse-ai/spec/models/ai_summary_spec.rb
+++ b/plugins/discourse-ai/spec/models/ai_summary_spec.rb
@@ -8,6 +8,120 @@ RSpec.describe AiSummary do
before { enable_current_plugin }
+ it "persists unsupported content languages using a supported cooking locale" do
+ SiteSetting.ai_summaries_for_crawlers = false
+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic, locale: "hi")
+ summary =
+ described_class.store!(strategy, llm_model, "**सारांश**", strategy.targets_data, human: true)
+
+ expect(summary.locale).to eq("hi")
+ expect(summary.summarized_cooked).to include("<strong>सारांश</strong>")
+
+ summary.update!(summarized_text: "Updated **सारांश**")
+ expect(summary.reload.summarized_cooked).to include("Updated <strong>सारांश</strong>")
+
+ summary.update_columns(summarized_cooked: nil)
+ timestamp = summary.updated_at
+ summary.cook_missing!
+ expect(summary.reload.summarized_cooked).to include("Updated <strong>सारांश</strong>")
+ expect(summary.locale).to eq("hi")
+ expect(summary.updated_at).to eq_time(timestamp)
+ end
+
+ it "stores and updates gist text without cooking HTML" do
+ strategy = DiscourseAi::Summarization::Strategies::HotTopicGists.new(topic, locale: "en")
+ summary =
+ described_class.store!(strategy, llm_model, "**Gist**", strategy.targets_data, human: true)
+ expect(summary.summarized_cooked).to be_nil
+
+ summary =
+ described_class.store!(
+ strategy,
+ llm_model,
+ "**Revised gist**",
+ strategy.targets_data,
+ human: true,
+ )
+ expect(summary.summarized_text).to eq("**Revised gist**")
+ expect(summary.summarized_cooked).to be_nil
+
+ summary.update!(summarized_text: "**Edited gist**")
+ summary.cook_missing!
+ expect(summary.reload.summarized_cooked).to be_nil
+
+ summary.update!(summary_type: :complete)
+ expect(summary.reload.summarized_cooked).to include("<strong>Edited gist</strong>")
+ summary.update!(summary_type: :gist)
+ expect(summary.reload.summarized_cooked).to be_nil
+ end
+
+ it "stores sanitized cooked content and replaces it when regenerating a summary" do
+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic, locale: "en")
+ summary =
+ described_class.store!(
+ strategy,
+ llm_model,
+ "**Overview** <script>alert(1)</script>",
+ strategy.targets_data,
+ human: true,
+ )
+
+ expect(summary.summarized_cooked).to include("<strong>Overview</strong>")
+ expect(summary.summarized_cooked).not_to include("<script>")
+
+ regenerated =
+ described_class.store!(
+ strategy,
+ llm_model,
+ "A revised **summary**",
+ strategy.targets_data,
+ human: true,
+ )
+
+ expect(regenerated.id).to eq(summary.id)
+ expect(regenerated.summarized_cooked).to include("A revised <strong>summary</strong>")
+ expect(regenerated.summarized_cooked).not_to include("Overview")
+ end
+
+ it "recooks changed text on ordinary saves" do
+ summary = Fabricate(:ai_summary, target: topic)
+ summary.update!(summarized_text: "Updated **text**")
+
+ expect(summary.reload.summarized_cooked).to include("Updated <strong>text</strong>")
+ end
+
+ it "cooks missing HTML without changing the summary freshness timestamp" do
+ summary = Fabricate(:ai_summary, target: topic, summarized_text: "**Original**")
+ summary.update_columns(summarized_cooked: nil)
+ updated_at = summary.updated_at
+
+ summary.cook_missing!
+
+ expect(summary.reload.summarized_cooked).to include("<strong>Original</strong>")
+ expect(summary.updated_at).to eq_time(updated_at)
+ end
+
+ it "does not overwrite a regenerated summary when cooking an older snapshot" do
+ summary = Fabricate(:ai_summary, target: topic, summarized_text: "Old summary")
+ summary.update_columns(summarized_cooked: nil)
+ snapshot = described_class.find(summary.id)
+ summary.update!(summarized_text: "New **summary**")
+
+ snapshot.cook_missing!
+
+ expect(summary.reload.summarized_cooked).to include("New <strong>summary</strong>")
+ expect(summary.summarized_text).to eq("New **summary**")
+ end
+
+ it "leaves existing cooked HTML alone" do
+ summary = Fabricate(:ai_summary, target: topic)
+ summary.update_columns(summarized_cooked: "<p>Previously cooked</p>")
+
+ summary.cook_missing!
+
+ expect(summary.reload.summarized_cooked).to eq("<p>Previously cooked</p>")
+ end
+
it "stores and independently upserts topic gists by locale" do
english_strategy =
DiscourseAi::Summarization::Strategies::HotTopicGists.new(topic, locale: "en")
diff --git a/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb b/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb
new file mode 100644
index 00000000000..6a05a5709c5
--- /dev/null
+++ b/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb
@@ -0,0 +1,475 @@
+# frozen_string_literal: true
+
+describe TopicsController do
+ fab!(:topic)
+ fab!(:first_post) { Fabricate(:post, topic: topic, post_number: 1) }
+
+ let(:summary_phrase) { "concise overview" }
+ let(:summary_text) { "A **#{summary_phrase}** of this discussion." }
+ let(:crawler_headers) { { "User-Agent" => "Googlebot" } }
+ let(:browser_headers) { { "User-Agent" => "Mozilla/5.0 Chrome/130.0.0.0 Safari/537.36" } }
+
+ before do
+ enable_current_plugin
+ assign_fake_provider_to(:ai_default_llm_model)
+ assign_agent_to(:ai_summarization_agent, [Group::AUTO_GROUPS[:admins]])
+ SiteSetting.ai_summarization_enabled = true
+ SiteSetting.enable_markdown_endpoints = true
+ SiteSetting.ai_summaries_for_crawlers = true
+ end
+
+ def create_summary(locale: "en", text: summary_text)
+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic, locale:)
+ Fabricate(
+ :ai_summary,
+ target: topic,
+ locale:,
+ summarized_text: text,
+ original_content_sha: strategy.summary_fingerprint[:original_content_sha],
+ highest_target_number: topic.highest_post_number,
+ )
+ end
+
+ describe "#show" do
+ it "keeps forced crawler translations consistent on misses and shared cache hits" do
+ global_setting :anon_cache_store_threshold, 1
+ Middleware::AnonymousCache.enable_anon_cache
+ SiteSetting.content_localization_enabled = true
+ SiteSetting.content_localization_supported_locales = "fr"
+ SiteSetting.default_locale = "fr"
+ topic.update!(locale: "en")
+ first_post.update!(locale: "en")
+ Fabricate(:post_localization, post: first_post, locale: "fr", cooked: "<p>Bonjour</p>")
+ create_summary
+ french = create_summary(locale: "fr", text: "Un résumé de cette discussion.")
+ forced_headers = browser_headers.merge("Discourse-Render" => "crawler")
+ bot_headers = { "User-Agent" => "#{browser_headers["User-Agent"]} Googlebot" }
+
+ %w[true false].each do |translate|
+ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = translate
+ [[forced_headers, bot_headers], [bot_headers, forced_headers]].each do |headers|
+ Middleware::AnonymousCache.clear_all_cache!
+ headers.each_with_index do |request_headers, index|
+ get topic.relative_url, headers: request_headers
+
+ expect(response.status).to eq(200)
+ expect(response.headers["X-Discourse-Cached"]).to eq(index.zero? ? "store" : "true")
+ document = Nokogiri.HTML5(response.body)
+ expect(document.at_css('[itemprop="abstract"]').text).to include(
+ translate == "true" ? french.summarized_text : summary_phrase,
+ )
+ expect(document.at_css("#post_1 .post").text).to include(
+ translate == "true" ? "Bonjour" : first_post.raw,
+ )
+ end
+ end
+ end
+ end
+
+ it "publishes a cached summary as the discussion abstract in crawler HTML" do
+ create_summary
+
+ get topic.relative_url, headers: crawler_headers
+
+ expect(response.status).to eq(200)
+ document = Nokogiri.HTML5(response.body)
+ abstract =
+ document.at_css(
+ '[itemtype="http://schema.org/DiscussionForumPosting"] > section [itemprop="abstract"]',
+ )
+ expect(abstract&.text).to include("A #{summary_phrase} of this discussion.")
+ expect(abstract.at_css("strong").text).to eq(summary_phrase)
+ expect(document.at_css("#post_1 .post").text).to include(first_post.raw)
+ end
+
+ it "leaves crawler Markdown responses unchanged" do
+ create_summary
+
+ get "#{topic.relative_url}.md", headers: crawler_headers
+
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include("## AI-generated summary", summary_phrase)
+ expect(response.body).to include(first_post.raw)
+ end
+
+ it "uses the same crawler classification as the anonymous cache" do
+ global_setting :anon_cache_store_threshold, 1
+ Middleware::AnonymousCache.enable_anon_cache
+ Middleware::AnonymousCache.clear_all_cache!
+ create_summary
+
+ get topic.relative_url, headers: browser_headers.merge("Discourse-Render" => "crawler")
+
+ expect(response.headers["X-Discourse-Cached"]).to eq("store")
+ expect(response.body).to include(summary_phrase)
+
+ get topic.relative_url,
+ headers: {
+ "User-Agent" => "#{browser_headers["User-Agent"]} Googlebot",
+ }
+
+ expect(response.headers["X-Discourse-Cached"]).to eq("true")
+ expect(response.body).to include(summary_phrase)
+ end
+
+ context "with HTML publication" do
+ def request_topic(path: topic.relative_url, params: {})
+ get path, params:, headers: crawler_headers
+ end
+
+ it "serves stored cooked content rather than cooking the source on request" do
+ summary = create_summary
+ stored_content = "Previously cooked overview"
+ summary.update_columns(summarized_cooked: "<p>#{stored_content}</p>")
+
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(stored_content)
+ expect(response.body).not_to include(summary_phrase)
+ end
+
+ it "omits uncooked summaries without cooking them on request" do
+ summary = create_summary
+ summary.update_columns(summarized_cooked: nil)
+
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(first_post.raw)
+ expect(response.body).not_to include(summary_phrase)
+ expect(summary.reload.summarized_cooked).to be_nil
+
+ summary.update_columns(summarized_cooked: "<p>#{summary_phrase}</p>")
+ request_topic
+
+ expect(response.body).to include(summary_phrase)
+ end
+
+ it "leaves browser and print responses unchanged" do
+ create_summary
+ path = topic.relative_url
+
+ [{}, { print: true }].each do |params|
+ SiteSetting.ai_summaries_for_crawlers = false
+ get path, params: params, headers: browser_headers
+ original_last_modified = response.headers["Last-Modified"]
+ SiteSetting.ai_summaries_for_crawlers = true
+
+ get path, params: params, headers: browser_headers
+
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary_phrase)
+ expect(response.headers["Last-Modified"]).to eq(original_last_modified)
+ end
+ end
+
+ it "adds one summary lookup per request without retaining it across requests" do
+ create_summary
+ SiteSetting.ai_summaries_for_crawlers = false
+ baseline_queries = track_sql_queries { request_topic }
+ SiteSetting.ai_summaries_for_crawlers = true
+
+ queries = track_sql_queries { request_topic }
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(summary_phrase)
+ summary_query = /SELECT.*FROM "ai_summaries"/
+ expect(queries.grep(summary_query).size).to eq(
+ baseline_queries.grep(summary_query).size + 1,
+ )
+
+ AiSummary.where(target: topic).delete_all
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary_phrase)
+
+ create_summary
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(summary_phrase)
+ end
+
+ it "serves summaries through the existing anonymous topic cache" do
+ global_setting :anon_cache_store_threshold, 1
+ Middleware::AnonymousCache.enable_anon_cache
+ Middleware::AnonymousCache.clear_all_cache!
+ summary = create_summary
+
+ request_topic
+
+ expect(response.headers["X-Discourse-Cached"]).to eq("store")
+ expect(request.env["ANON_CACHE_DURATION"]).to eq(1.minute)
+ expect(response.body).to include(summary_phrase)
+
+ path = topic.relative_url
+ get path, headers: browser_headers
+ expect(response.headers["X-Discourse-Cached"]).to eq("store")
+ expect(response.body).not_to include(summary_phrase)
+
+ get path, headers: browser_headers
+ expect(response.headers["X-Discourse-Cached"]).to eq("true")
+ expect(response.body).not_to include(summary_phrase)
+
+ first_post.update!(last_version_at: summary.updated_at + 1.minute)
+ queries = track_sql_queries { request_topic }
+
+ expect(response.headers["X-Discourse-Cached"]).to eq("true")
+ expect(response.body).to include(summary_phrase)
+ expect(queries.grep(/\b(?:ai_summaries|posts|topics)\b/)).to be_empty
+
+ Middleware::AnonymousCache.clear_all_cache!
+ request_topic
+
+ expect(response.headers["X-Discourse-Cached"]).to eq("store")
+ expect(response.body).not_to include(summary_phrase)
+
+ request_topic
+
+ expect(response.headers["X-Discourse-Cached"]).to eq("true")
+ expect(response.body).not_to include(summary_phrase)
+ end
+
+ it "omits summaries when publication, summarization, or the plugin is disabled" do
+ summary = create_summary
+
+ %i[
+ ai_summaries_for_crawlers
+ ai_summarization_enabled
+ discourse_ai_enabled
+ ].each do |setting|
+ SiteSetting.public_send("#{setting}=", false)
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary.summarized_text, summary_phrase)
+ SiteSetting.public_send("#{setting}=", true)
+ end
+ end
+
+ it "leaves a missing summary absent without generating one" do
+ DiscourseAi::Completions::Llm.with_prepared_responses([]) do
+ expect { request_topic }.not_to change(AiSummary, :count)
+ end
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(first_post.raw)
+ expect(response.body).not_to include(
+ I18n.t("discourse_ai.summarization.published_summary_heading"),
+ )
+ expect(response.headers["Last-Modified"]).to eq(first_post.updated_at.httpdate)
+ expect(request.env["ANON_CACHE_DURATION"]).to eq(1.minute)
+ end
+
+ it "omits an edited summary even for an admin and preserves the post freshness hint" do
+ summary = create_summary
+ sign_in(Fabricate(:admin))
+ request_topic
+ expect(response.body).to include(summary_phrase)
+
+ first_post.update!(last_version_at: summary.updated_at + 1.minute)
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary_phrase)
+ expect(response.headers["Last-Modified"]).to eq(first_post.reload.updated_at.httpdate)
+ end
+
+ it "omits a summary after new replies or hidden source posts change the input" do
+ summary = create_summary
+ reply = Fabricate(:post, topic:, post_number: 2)
+ topic.update!(highest_post_number: 2)
+ request_topic
+ expect(response.body).not_to include(summary_phrase)
+
+ summary.destroy!
+ create_summary
+ request_topic
+ expect(response.body).to include(summary_phrase)
+
+ reply.update!(hidden: true)
+ request_topic
+ expect(response.body).not_to include(summary_phrase)
+ end
+
+ it "omits a summary after a source post is deleted" do
+ reply = Fabricate(:post, topic:, post_number: 2)
+ topic.update!(highest_post_number: 2)
+ create_summary
+ request_topic
+ expect(response.body).to include(summary_phrase)
+
+ reply.trash!
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary_phrase)
+ expect(response.headers["Last-Modified"]).to eq(first_post.reload.updated_at.httpdate)
+ end
+
+ it "omits restricted topics even when the requester is an admin" do
+ create_summary
+ topic.update!(category: Fabricate(:private_category, group: Fabricate(:group)))
+ sign_in(Fabricate(:admin))
+
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(first_post.raw)
+ expect(response.body).not_to include(summary_phrase)
+ end
+
+ it "omits summaries when the site requires login" do
+ create_summary
+ SiteSetting.login_required = true
+ sign_in(Fabricate(:admin))
+
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary_phrase)
+ end
+
+ it "omits personal messages even when their participant can summarize" do
+ admin = Fabricate(:admin)
+ message = Fabricate(:private_message_topic, user: admin)
+ message_post = Fabricate(:post, topic: message)
+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(message, locale: "en")
+ Fabricate(
+ :ai_summary,
+ target: message,
+ locale: "en",
+ summarized_text: summary_text,
+ original_content_sha: strategy.summary_fingerprint[:original_content_sha],
+ )
+ SiteSetting.ai_pm_summarization_allowed_groups = Group::AUTO_GROUPS[:admins].to_s
+ sign_in(admin)
+
+ request_topic(path: message.relative_url)
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(message_post.raw)
+ expect(response.body).not_to include(summary_phrase)
+ end
+
+ it "omits summaries from later pages and single-post responses" do
+ Fabricate.times(TopicView.chunk_size, :post, topic:)
+ topic.update!(highest_post_number: TopicView.chunk_size + 1)
+ create_summary
+
+ request_topic(params: { page: 2 })
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary_phrase)
+
+ request_topic(path: "#{topic.relative_url}/2")
+ expect(response.status).to eq(200)
+ expect(response.body).not_to include(summary_phrase)
+ end
+
+ it "leaves Last-Modified and topic activity unchanged when a summary is regenerated" do
+ freeze_time
+ topic.update!(last_posted_at: first_post.created_at)
+ summary = create_summary
+ request_topic
+ previous_last_modified = response.headers["Last-Modified"]
+ topic_activity = topic.reload.last_posted_at
+
+ freeze_time 2.minutes.from_now
+ summary.update!(summarized_text: "An updated overview")
+ request_topic
+
+ expect(response.body).to include(summary.summarized_text)
+ expect(response.headers["Last-Modified"]).to eq(previous_last_modified)
+ expect(topic.reload.last_posted_at).to eq_time(topic_activity)
+ expect(request.env["ANON_CACHE_DURATION"]).to eq(1.minute)
+ end
+
+ it "uses the source-language summary instead of another cached locale" do
+ topic.update!(locale: "fr")
+ english = create_summary
+ french = create_summary(locale: "fr", text: "Un résumé de cette discussion.")
+
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(french.summarized_text)
+ expect(response.body).not_to include(english.summarized_text, summary_phrase)
+ end
+
+ it "uses a localized summary when the discussion is shown translated" do
+ SiteSetting.content_localization_enabled = true
+ SiteSetting.content_localization_supported_locales = "fr"
+ SiteSetting.default_locale = "fr"
+ topic.update!(locale: "en")
+ first_post.update!(locale: "en")
+ Fabricate(:post_localization, post: first_post, locale: "fr", cooked: "<p>Bonjour</p>")
+ english = create_summary
+ french = create_summary(locale: "fr", text: "Un résumé de cette discussion.")
+
+ request_topic
+
+ expect(response.status).to eq(200)
+ expect(response.body).to include(french.summarized_text)
+ expect(response.body).not_to include(english.summarized_text, summary_phrase)
+
+ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = "false"
+ request_topic
+
+ expect(response.body).to include(summary_phrase, first_post.raw)
+ expect(response.body).not_to include(french.summarized_text, "Bonjour")
+
+ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = "true"
+ french.destroy!
+ request_topic
+ expect(response.body).not_to include(
+ english.summarized_text,
+ french.summarized_text,
+ summary_phrase,
+ )
+ end
+ end
+
+ it "sanitizes summary markup in HTML" do
+ create_summary(
+ text: 'Overview <script>alert("xss")</script> <a href="javascript:alert(1)">unsafe</a>',
+ )
+
+ get topic.relative_url, headers: crawler_headers
+ abstract = Nokogiri.HTML5(response.body).at_css('[itemprop="abstract"]')
+ expect(abstract.text).to include("Overview")
+ expect(abstract.css("script, [href^='javascript:']")).to be_empty
+ end
+
+ it "leaves topic JSON unchanged" do
+ summary = create_summary
+ summary.update!(updated_at: first_post.updated_at + 1.minute)
+
+ get "#{topic.relative_url}.json", headers: crawler_headers
+
+ expect(response.status).to eq(200)
+ expect(response.parsed_body["has_cached_summary"]).to eq(true)
+ expect(response.parsed_body).not_to have_key("ai_summary")
+ expect(response.headers["Last-Modified"]).to be_nil
+ end
+
+ it "keeps the abstract on QAPage rather than the nested question or answers" do
+ SiteSetting.allow_solved_on_all_topics = true
+ SiteSetting.solved_add_schema_markup = "always"
+ Fabricate(:post, topic:, post_number: 2)
+ topic.update!(highest_post_number: 2)
+ create_summary
+
+ get topic.relative_url, headers: crawler_headers
+
+ expect(response.status).to eq(200)
+ document = Nokogiri.HTML5(response.body)
+ abstract = document.at_css('[itemprop="abstract"]')
+ expect(abstract.ancestors("[itemscope]").first["itemtype"]).to eq("https://schema.org/QAPage")
+ expect(
+ document.css('[itemtype="https://schema.org/Question"] [itemprop="abstract"]'),
+ ).to be_empty
+ end
+ end
+end
diff --git a/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.js b/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.js
index 4ce7d184809..a1e94a2c3e0 100644
--- a/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.js
+++ b/plugins/discourse-ai/test/javascripts/unit/lib/ai-feature-setting-groups-test.js
@@ -66,6 +66,10 @@ module("Unit | Lib | ai-feature-setting-groups", function () {
const groups = getSettingGroupsForFeature("summarization");
assert.strictEqual(groups.length, 3, "summarization should have 3 groups");
+ assert.true(
+ groups[0].settings.includes("ai_summaries_for_crawlers"),
+ "basic settings should include summary publication"
+ );
});
test("returns correct groups for search", function (assert) {
diff --git a/spec/requests/topics_controller_spec.rb b/spec/requests/topics_controller_spec.rb
index ff9c7c0f4c1..efbaa14d8d4 100644
--- a/spec/requests/topics_controller_spec.rb
+++ b/spec/requests/topics_controller_spec.rb
@@ -7943,6 +7943,29 @@ RSpec.describe TopicsController do
expect(response.body).to include(subcategory.name)
expect(response.body).to include(topic.first_post.cooked)
end
+
+ it "localizes forced crawler HTML while respecting translation opt-out" do
+ SiteSetting.default_locale = "pt"
+ headers = {
+ "User-Agent" => "Mozilla/5.0 Chrome/130.0.0.0 Safari/537.36",
+ "Discourse-Render" => "crawler",
+ }
+
+ get topic.relative_url, headers: headers
+
+ document = Nokogiri.HTML5(response.body)
+ expect(document.at_css("#post_1 .post").text).to include(
+ Nokogiri::HTML5.fragment(pt_first_post.cooked).text,
+ )
+
+ cookies[ContentLocalization::AUTOMATICALLY_TRANSLATE_COOKIE] = "false"
+ get topic.relative_url, headers: headers
+
+ document = Nokogiri.HTML5(response.body)
+ expect(document.at_css("#post_1 .post").text).to include(
+ Nokogiri::HTML5.fragment(topic.first_post.cooked).text,
+ )
+ end
end
describe "when tl param is present ?tl=ja" do