A platform for community discussion. Free, open, simple.
FEATURE: Include AI summaries in Markdown topic responses (#44360)
Natalie Taydf46f8a7af5e
diff --git a/lib/markdown_endpoint/topic_renderer.rb b/lib/markdown_endpoint/topic_renderer.rbindex d5bfc0a9a54..3c3dcf14eef 100644--- a/lib/markdown_endpoint/topic_renderer.rb+++ b/lib/markdown_endpoint/topic_renderer.rb@@ -18,6 +18,11 @@ module MarkdownEndpointraise Discourse::NotFound if single_post? && posts.empty?buffer = String.new(header(posts.length))+ unless single_post?+ DiscoursePluginRegistry+ .apply_modifier(:markdown_topic_header_sections, [], @topic_view, @guardian)+ .each { |section| buffer << "\n\n" << section }+ endposts.each_with_index do |post, index|buffer << (index.zero? ? "\n\n" : "\n\n---\n\n")buffer << render_post(post)diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/summarization/summary_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/summarization/summary_controller.rbindex 8995dd6883f..365f4857f39 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/summarization/summary_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/summarization/summary_controller.rb@@ -12,8 +12,6 @@ module DiscourseAiguardian.ensure_can_see!(topic)summarization_service =DiscourseAi::TopicSummarization.for(topic, current_user, scope: guardian)- raise Discourse::NotFound if !summarization_service.available?-cached_summary = summarization_service.cached_summaryraise Discourse::NotFound if !cached_summarydiff --git a/plugins/discourse-ai/app/services/discourse_ai/topic_summarization.rb b/plugins/discourse-ai/app/services/discourse_ai/topic_summarization.rbindex 0457c1b0ee6..6b11cf91a52 100644--- a/plugins/discourse-ai/app/services/discourse_ai/topic_summarization.rb+++ b/plugins/discourse-ai/app/services/discourse_ai/topic_summarization.rb@@ -6,12 +6,19 @@ module DiscourseAidef self.for(topic, user, scope: nil, locale: nil)scope ||= user&.guardian || Guardian.newlocale ||= DiscourseAi::Summarization.display_locale(topic, scope:)- new(DiscourseAi::Summarization.topic_summary(topic, locale:), user)+ cached_reader =+ if SiteSetting.ai_summarization_enabled+ strategy = DiscourseAi::Summarization::Strategies::TopicSummary.new(topic, locale:)+ DiscourseAi::Summarization::FoldContent.new(strategy)+ end++ new(cached_reader, user) { DiscourseAi::Summarization.topic_summary(topic, locale:) }end- def initialize(summarizer, user)+ def initialize(summarizer, user, &build_generator)@summarizer = summarizer@user = user+ @build_generator = build_generatorenddef cached_summary@@ -21,7 +28,7 @@ module DiscourseAienddef available?- summarizer.present?+ generator.present?enddef locale@@ -37,13 +44,19 @@ module DiscourseAireturn cached_summary if !force_regenerate && use_cached?(skip_age_check)- summarizer.summarize(user, &on_partial_blk)+ generator&.summarize(user, &on_partial_blk)endprivateattr_reader :summarizer, :user+ def generator+ return @generator if defined?(@generator)++ @generator = @build_generator ? @build_generator.call : summarizer+ end+def use_cached?(skip_age_check)return false if !cached_summarydiff --git a/plugins/discourse-ai/config/locales/server.en.yml b/plugins/discourse-ai/config/locales/server.en.ymlindex 6cc5313ae5f..4c0415cee90 100644--- a/plugins/discourse-ai/config/locales/server.en.yml+++ b/plugins/discourse-ai/config/locales/server.en.yml@@ -147,7 +147,7 @@ en:ai_admin_dashboard_highlights_category_scope: "Choose which categories AI highlights can use. Include and exclude options use {{setting:ai_admin_dashboard_highlights_categories}}. Personal messages are always excluded."ai_admin_dashboard_highlights_categories: "Categories used when the AI highlights category scope is set to <strong>include</strong> or <strong>exclude</strong>."ai_summarization_enabled: "Enable the summarize feature"- ai_summaries_for_crawlers: "Include AI topic summaries in public HTML topic pages served to search engines and other crawlers. Requires {{setting:ai_summarization_enabled}} and {{setting:ai_summary_backfill_maximum_topics_per_hour}} above 0, so summaries stay up to date."+ ai_summaries_for_crawlers: "Include AI topic summaries in public HTML topic pages served to search engines and other crawlers, and in the Markdown version of those pages. Requires {{setting:ai_summarization_enabled}} and {{setting:ai_summary_backfill_maximum_topics_per_hour}} above 0, so summaries stay up to date."ai_summarization_agent: "Agent to use for summarize feature"ai_pm_summarization_allowed_groups: "Groups allowed to create and view summaries in PMs."ai_summary_gists_enabled: "Generate brief summaries of latest replies in topics automatically"diff --git a/plugins/discourse-ai/lib/summarization/entry_point.rb b/plugins/discourse-ai/lib/summarization/entry_point.rbindex 7d84072d8a1..280325291a3 100644--- a/plugins/discourse-ai/lib/summarization/entry_point.rb+++ b/plugins/discourse-ai/lib/summarization/entry_point.rb@@ -10,7 +10,7 @@ module DiscourseAitopic_view,guardian: controller.guardian,)- next if !publication.summary+ next if !publication.summary&.summarized_cooked.present?controller.render_to_string(partial: "discourse_ai/summarization/published_summary",@@ -20,6 +20,21 @@ module DiscourseAi)end+ # markdown is a machine format, so agents get the summary whatever their user agent+ plugin.register_modifier(+ :markdown_topic_header_sections,+ ) do |sections, topic_view, guardian|+ publication =+ DiscourseAi::Summarization::PublishedSummary.new(+ topic_view,+ guardian:,+ crawler_only: false,+ )+ next sections if !publication.summary++ sections + [publication.markdown]+ end+plugin.add_to_serializer(:current_user, :can_request_gists) { scope.can_request_gists? }plugin.add_to_serializer(:current_user, :can_summarize) dodiff --git a/plugins/discourse-ai/lib/summarization/fold_content.rb b/plugins/discourse-ai/lib/summarization/fold_content.rbindex fee3199e09e..146ae48f05f 100644--- a/plugins/discourse-ai/lib/summarization/fold_content.rb+++ b/plugins/discourse-ai/lib/summarization/fold_content.rb@@ -43,14 +43,15 @@ module DiscourseAi# Finds a summary matching the target and strategy. Marks it as outdated if the strategy found newer contentdef existing_summaryif !defined?(@existing_summary)- summaries = AiSummary.where(target: strategy.target, summary_type: strategy.type)- summary = summaries.find_by(locale: strategy.locale)+ summaries = AiSummary.where(target: strategy.target, summary_type: strategy.type).to_a+ summary = summaries.find { |candidate| candidate.locale == strategy.locale }if summary.blank? && strategy.locale.present?summary =- summaries- .where.not(locale: nil)- .find { |candidate| LocaleNormalizer.is_same?(candidate.locale, strategy.locale) }+ summaries.find do |candidate|+ candidate.locale.present? &&+ LocaleNormalizer.is_same?(candidate.locale, strategy.locale)+ endendif summarydiff --git a/plugins/discourse-ai/lib/summarization/published_summary.rb b/plugins/discourse-ai/lib/summarization/published_summary.rbindex c5031669e22..6d22860243a 100644--- a/plugins/discourse-ai/lib/summarization/published_summary.rb+++ b/plugins/discourse-ai/lib/summarization/published_summary.rb@@ -3,14 +3,14 @@module DiscourseAimodule Summarizationclass PublishedSummary- def initialize(topic_view, guardian:)+ def initialize(topic_view, guardian:, crawler_only: true)@topic_view = topic_view@guardian = guardian+ @crawler_only = crawler_onlyenddef eligible?- request = @guardian.request- return false if !request || !Middleware::AnonymousCache::Helper.new(request.env).is_crawler?+ return false if @crawler_only && !crawler_request?topic = @topic_view.topicSiteSetting.ai_summaries_for_crawlers && SiteSetting.ai_summarization_enabled &&@@ -31,7 +31,7 @@ module DiscourseAinil,scope: @guardian,).cached_summary- if cached_summary&.summarized_cooked.present? &&+ if cached_summary&.summarized_text.present? &&Guardian.new.can_see_summary?(@topic_view.topic, cached_summary:)@summary = cached_summaryend@@ -41,6 +41,18 @@ module DiscourseAidef cookedsummary.summarized_cookedend++ def markdown+ heading = I18n.t("discourse_ai.summarization.published_summary_heading")+ "## #{heading}\n\n#{summary.summarized_text}"+ end++ private++ def crawler_request?+ request = @guardian.request+ request.present? && Middleware::AnonymousCache::Helper.new(request.env).is_crawler?+ endendendenddiff --git a/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb b/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rbindex 4a5b4ca8060..d8e2acc0182 100644--- a/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb+++ b/plugins/discourse-ai/spec/requests/summarization/published_summary_spec.rb@@ -87,14 +87,79 @@ describe TopicsController doexpect(document.at_css("#post_1 .post").text).to include(first_post.raw)end- it "leaves crawler Markdown responses unchanged" do+ it "adds the summary before the posts in Markdown for crawlers and agents" docreate_summary+ heading = "## #{I18n.t("discourse_ai.summarization.published_summary_heading")}"++ [+ ["#{topic.relative_url}.md", crawler_headers],+ ["#{topic.relative_url}.md", browser_headers],+ [topic.relative_url, browser_headers.merge("Accept" => "text/markdown")],+ ].each do |path, headers|+ get path, headers: headers++ expect(response.status).to eq(200)+ body = response.body+ expect(body).to include("#{heading}\n\nA **#{summary_phrase}** of this discussion.")+ expect(body.index(heading)).to be < body.index(first_post.raw)+ end+ end++ it "publishes the stored Markdown without requiring cooked content" do+ text = "An overview of 猫.\n\n[Reference][source]\n\n[source]: https://example.com"+ summary = create_summary(text:)++ get "#{topic.relative_url}.md", headers: browser_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(text)++ summary.update_columns(summarized_cooked: nil)++ get "#{topic.relative_url}.md", headers: browser_headers++ expect(response.status).to eq(200)+ expect(response.body).to include(text)+ expect(summary.reload.summarized_cooked).to be_nil+ end++ it "reads the Markdown summary with one query and no LLM lookup whether or not one exists" do+ path = "#{topic.relative_url}.md"++ queries = track_sql_queries { get path, headers: browser_headers }++ expect(response.status).to eq(200)+ expect(queries.grep(/FROM "ai_summaries"/).size).to eq(1)++ create_summary+ queries = track_sql_queries { get path, headers: browser_headers }++ expect(response.body).to include(summary_phrase)+ expect(queries.grep(/FROM "ai_summaries"/).size).to eq(1)+ expect(queries.grep(/FROM "llm_models"/)).to be_empty+ end++ it "leaves the summary out of later Markdown pages and single posts" do+ 2.upto(TopicView::CHUNK_SIZE + 1) { |post_number| Fabricate(:post, topic:, post_number:) }+ topic.update!(highest_post_number: TopicView::CHUNK_SIZE + 1)+ create_summary++ ["#{topic.relative_url}.md?page=2", "#{topic.relative_url}/1.md"].each do |path|+ get path, headers: crawler_headers++ expect(response.status).to eq(200)+ expect(response.body).not_to include(summary_phrase)+ end+ end++ it "leaves an outdated summary out of Markdown" do+ summary = create_summary+ first_post.update!(last_version_at: summary.updated_at + 1.minute)get "#{topic.relative_url}.md", headers: crawler_headersexpect(response.status).to eq(200)- expect(response.body).not_to include("## AI-generated summary", summary_phrase)- expect(response.body).to include(first_post.raw)+ expect(response.body).not_to include(summary_phrase)endit "uses the same crawler classification as the anonymous cache" dodiff --git a/plugins/discourse-ai/spec/requests/summarization/summary_controller_spec.rb b/plugins/discourse-ai/spec/requests/summarization/summary_controller_spec.rbindex c23cc194138..a4f4b2cd7b7 100644--- a/plugins/discourse-ai/spec/requests/summarization/summary_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/summarization/summary_controller_spec.rb@@ -100,9 +100,10 @@ RSpec.describe DiscourseAi::Summarization::SummaryController doit "returns a fresh cached summary" dosummary = create_cached_summary(topic)- get "/discourse-ai/summarization/t/#{topic.id}.json"+ queries = track_sql_queries { get "/discourse-ai/summarization/t/#{topic.id}.json" }expect(response.status).to eq(200)+ expect(queries.grep(/FROM "llm_models"/)).to be_emptyresponse_summary = response.parsed_bodyexpect(response_summary.dig("ai_topic_summary", "summarized_text")).to eq(diff --git a/plugins/discourse-ai/spec/services/discourse_ai/topic_summarization_spec.rb b/plugins/discourse-ai/spec/services/discourse_ai/topic_summarization_spec.rbindex 97df835a52e..1d08d63698c 100644--- a/plugins/discourse-ai/spec/services/discourse_ai/topic_summarization_spec.rb+++ b/plugins/discourse-ai/spec/services/discourse_ai/topic_summarization_spec.rb@@ -56,6 +56,45 @@ describe DiscourseAi::TopicSummarization doexpect(described_class.for(topic, nil).cached_summary).to eq(cached_summary)expect(DiscourseAi::Summarization).not_to have_received(:build_bot)end++ it "reads a cached summary without querying LLM models" do+ cached_summary = create_cached_summary(topic)+ result = nil++ queries = track_sql_queries { result = described_class.for(topic, nil).cached_summary }++ expect(result).to eq(cached_summary)+ expect(queries.grep(/FROM "llm_models"/)).to be_empty+ end++ it "can read a stored summary when generation is unavailable" do+ cached_summary = create_cached_summary(topic)+ SiteSetting.ai_summarization_agent = ""+ service = described_class.for(topic, user)++ expect(service.cached_summary).to eq(cached_summary)+ expect(service).not_to be_available+ expect(service.summarize(force_regenerate: true)).to be_nil+ end++ it "does not read or generate summaries when summarization is disabled" do+ create_cached_summary(topic)+ SiteSetting.ai_summarization_enabled = false+ service = described_class.for(topic, user)++ expect(service.cached_summary).to be_nil+ expect(service).not_to be_available+ expect(service.summarize).to be_nil+ end++ it "generates a summary through the shared topic API" do+ DiscourseAi::Completions::Llm.with_prepared_responses([summary]) do+ service = described_class.for(topic, user)++ expect(service).to be_available+ expect(service.summarize.summarized_text).to eq(summary)+ end+ endenddescribe "#summarize" dodiff --git a/spec/requests/markdown_endpoints_spec.rb b/spec/requests/markdown_endpoints_spec.rbindex a7f6d89b459..e0628301e29 100644--- a/spec/requests/markdown_endpoints_spec.rb+++ b/spec/requests/markdown_endpoints_spec.rb@@ -277,6 +277,40 @@ RSpec.describe "Markdown endpoints" doendend+ it "lets plugins add topic sections between the metadata and the first post" do+ plugin_instance = Plugin::Instance.new+ modifier =+ Proc.new do |sections, topic_view, guardian|+ sections + ["## Plugin section", "Topic #{topic_view.topic.id} for #{guardian.user&.id}"]+ end+ DiscoursePluginRegistry.register_modifier(+ plugin_instance,+ :markdown_topic_header_sections,+ &modifier+ )++ begin+ get "/t/#{topic.slug}/#{topic.id}.md"++ expect(response).to have_http_status(:ok)+ body = response.body+ expect(body).to include("**Page:** 1\n\n## Plugin section\n\nTopic #{topic.id} for \n\n")+ expect(body.index("## Plugin section")).to be < body.index(post.raw)++ get "/t/#{topic.slug}/#{topic.id}/1.md"++ expect(response).to have_http_status(:ok)+ expect(response.body).to include(post.raw)+ expect(response.body).not_to include("## Plugin section")+ ensure+ DiscoursePluginRegistry.unregister_modifier(+ plugin_instance,+ :markdown_topic_header_sections,+ &modifier+ )+ end+ end+it "paginates rendered posts with translated Markdown navigation links" doTranslationOverride.upsert!("en", "markdown_endpoints.previous_page", "Earlier posts")TranslationOverride.upsert!("en", "markdown_endpoints.next_page", "Later posts")