A platform for community discussion. Free, open, simple.
FIX: Stop inserting stray spaces into posts sent for AI summaries (#44047)
Régis Hanol5404705d76f4
diff --git a/plugins/discourse-ai/lib/summarization/fold_content.rb b/plugins/discourse-ai/lib/summarization/fold_content.rbindex f96a913a547..56db9a5730d 100644--- a/plugins/discourse-ai/lib/summarization/fold_content.rb+++ b/plugins/discourse-ai/lib/summarization/fold_content.rb@@ -3,11 +3,6 @@module DiscourseAimodule Summarization# This class offers a generic way of summarizing content from multiple sources using different prompts.- #- # It summarizes large amounts of content by recursively summarizing it in smaller chunks that- # fit the given model context window, finally concatenating the disjoint summaries- # into a final version.- #class FoldContentclass MissingToolOutput < StandardErrorend@@ -22,7 +17,6 @@ module DiscourseAi# @param user { User } - User object used for auditing usage.# @param &on_partial_blk { Block - Optional } - The passed block will get called with the LLM partial response.- # Note: The block is only called with results of the final summary, not intermediate summaries.## This method doesn't care if we already have an up to date summary. It always regenerate.#@@ -63,24 +57,13 @@ module DiscourseAi@existing_summaryend- def delete_cached_summaries!- summaries = AiSummary.where(target: strategy.target, summary_type: strategy.type)-- if strategy.locale.present?- summary_ids =- summaries- .where.not(locale: nil)- .filter_map do |summary|- summary.id if LocaleNormalizer.is_same?(summary.locale, strategy.locale)- end- AiSummary.where(id: summary_ids).destroy_all- else- summaries.where(locale: nil).destroy_all- end- end-def truncate(item)item_content = item[:text].to_s+ truncation_length = 500+ tokenizer = llm_model.tokenizer_class+ strict = SiteSetting.ai_strict_token_counting+ return item if tokenizer.below_limit?(item_content, truncation_length * 2, strict:)+# From https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries:## A single Unicode code point is often, but not always, the same as a basic unit of a@@ -94,24 +77,13 @@ module DiscourseAigraphemes = item_content.grapheme_clustersmidpoint = graphemes.size / 2- first_half = graphemes.slice(0, midpoint)&.join || ""- second_half = (graphemes.slice(midpoint, graphemes.size - midpoint) || []).join-- truncation_length = 500- tokenizer = llm_model.tokenizer_class+ first_half = graphemes[...midpoint].join+ second_half = graphemes[midpoint..].join- item[:text] = [- tokenizer.truncate(- first_half,- truncation_length,- strict: SiteSetting.ai_strict_token_counting,- ).to_s,- tokenizer.truncate(- second_half,- truncation_length,- strict: SiteSetting.ai_strict_token_counting,- ).to_s,- ].join(" ")+ head = tokenizer.truncate(first_half, truncation_length, strict:)+ tail_length = tokenizer.decode(tokenizer.encode(second_half).last(truncation_length)).length+ tail = second_half[(second_half.length - tail_length)..]+ item[:text] = "#{head} #{tail}"itemend@@ -147,10 +119,6 @@ module DiscourseAi# @param items { Array<Hash> } - Content to summarize. Structure will be: { poster: who wrote the content, id: a way to order content, text: content }# @param user { User } - User object used for auditing usage.# @param &on_partial_blk { Block - Optional } - The passed block will get called with the LLM partial response.- # Note: The block is only called with results of the final summary, not intermediate summaries.- #- # The summarization algorithm.- # It will summarize as much content summarize given the model's context window. If will prioriotize newer content in case it doesn't fit.## @returns { String } - Resulting summary.def fold(items, user, &on_partial_blk)@@ -158,7 +126,7 @@ module DiscourseAitokens_left = available_tokenscontent_in_window = []- items.each_with_index do |item, idx|+ items.each do |item|as_text = "(#{item[:id]} #{item[:poster]} said: #{item[:text]} "if tokenizer.below_limit?(diff --git a/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rb b/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rbindex ab4f9bfa103..d12f9b5feba 100644--- a/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rb@@ -237,7 +237,7 @@ RSpec.describe DiscourseAi::Summarization::FoldContent doexpect(regional_summarizer.existing_summary).to eq(existing_gist)end- it "finds and deletes a complete summary stored under an equivalent regional locale" do+ it "finds a complete summary stored under an equivalent regional locale" doexisting_summary =Fabricate(:ai_summary,@@ -249,9 +249,6 @@ RSpec.describe DiscourseAi::Summarization::FoldContent doregional_summarizer = DiscourseAi::Summarization.topic_summary(topic, locale: "pt_BR")expect(regional_summarizer.existing_summary).to eq(existing_summary)-- regional_summarizer.delete_cached_summaries!- expect(AiSummary.find_by(id: existing_summary.id)).to be_nilendcontext "when a summary already exists" do@@ -289,24 +286,30 @@ RSpec.describe DiscourseAi::Summarization::FoldContent doenddescribe "#truncate" do- it "preserves grapheme clusters for multi-codepoint emoji sequences" do- # Starts with scales emoji (⚖️ = U+2696 + U+FE0F) so we can catch any split between code points.- sample_text = "⚖️🧩"+ it "leaves content within the token limit untouched" do+ text = "用 [keyd](https://man.archlinux.org/man/extra/keyd/keyd.1.en) 改键"++ expect(summarizer.truncate({ text: text.dup })[:text]).to eq(text)+ end- item = summarizer.truncate({ text: sample_text.dup })+ it "doesn't let the tokenizer rewrite content within the token limit" do+ llm_model.update!(tokenizer: "DiscourseAi::Tokenizer::AnthropicTokenizer")+ text = "改键,很好用。" * 100- expect(item[:text]).to start_with("⚖️ ")- expect(item[:text]).to include("🧩")- expect(item[:text]).not_to start_with("⚖ ️")+ expect(summarizer.truncate({ text: text.dup })[:text]).to eq(text)end- it "keeps the second half of the text in the original order" do- sample_text = "abcdefgh"+ it "keeps the start and the end of longer content" do+ item = summarizer.truncate({ text: "start #{"a " * 1500}#{"b " * 1500}end" })++ expect(item[:text]).to start_with("start a")+ expect(item[:text]).to end_with("b end")+ end- item = summarizer.truncate({ text: sample_text.dup })+ it "preserves grapheme clusters at the split point" do+ item = summarizer.truncate({ text: "#{"🧩" * 600}⚖️#{"a" * 600}" })- expect(item[:text]).to include("efgh")- expect(item[:text]).not_to include("hgfe")+ expect(item[:text]).to include(" ⚖️a")endendend