Skip to content

A platform for community discussion. Free, open, simple.

FIX: Stop inserting stray spaces into posts sent for AI summaries (#44047)

Régis Hanol5404705d76f4
diff --git a/plugins/discourse-ai/lib/summarization/fold_content.rb b/plugins/discourse-ai/lib/summarization/fold_content.rb
index f96a913a547..56db9a5730d 100644
--- a/plugins/discourse-ai/lib/summarization/fold_content.rb
+++ b/plugins/discourse-ai/lib/summarization/fold_content.rb
@@ -3,11 +3,6 @@
module DiscourseAi
module Summarization
# This class offers a generic way of summarizing content from multiple sources using different prompts.
- #
- # It summarizes large amounts of content by recursively summarizing it in smaller chunks that
- # fit the given model context window, finally concatenating the disjoint summaries
- # into a final version.
- #
class FoldContent
class MissingToolOutput < StandardError
end
@@ -22,7 +17,6 @@ module DiscourseAi
# @param user { User } - User object used for auditing usage.
# @param &on_partial_blk { Block - Optional } - The passed block will get called with the LLM partial response.
- # Note: The block is only called with results of the final summary, not intermediate summaries.
#
# This method doesn't care if we already have an up to date summary. It always regenerate.
#
@@ -63,24 +57,13 @@ module DiscourseAi
@existing_summary
end
- def delete_cached_summaries!
- summaries = AiSummary.where(target: strategy.target, summary_type: strategy.type)
-
- if strategy.locale.present?
- summary_ids =
- summaries
- .where.not(locale: nil)
- .filter_map do |summary|
- summary.id if LocaleNormalizer.is_same?(summary.locale, strategy.locale)
- end
- AiSummary.where(id: summary_ids).destroy_all
- else
- summaries.where(locale: nil).destroy_all
- end
- end
-
def truncate(item)
item_content = item[:text].to_s
+ truncation_length = 500
+ tokenizer = llm_model.tokenizer_class
+ strict = SiteSetting.ai_strict_token_counting
+ return item if tokenizer.below_limit?(item_content, truncation_length * 2, strict:)
+
# From https://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries:
#
# A single Unicode code point is often, but not always, the same as a basic unit of a
@@ -94,24 +77,13 @@ module DiscourseAi
graphemes = item_content.grapheme_clusters
midpoint = graphemes.size / 2
- first_half = graphemes.slice(0, midpoint)&.join || ""
- second_half = (graphemes.slice(midpoint, graphemes.size - midpoint) || []).join
-
- truncation_length = 500
- tokenizer = llm_model.tokenizer_class
+ first_half = graphemes[...midpoint].join
+ second_half = graphemes[midpoint..].join
- item[:text] = [
- tokenizer.truncate(
- first_half,
- truncation_length,
- strict: SiteSetting.ai_strict_token_counting,
- ).to_s,
- tokenizer.truncate(
- second_half,
- truncation_length,
- strict: SiteSetting.ai_strict_token_counting,
- ).to_s,
- ].join(" ")
+ head = tokenizer.truncate(first_half, truncation_length, strict:)
+ tail_length = tokenizer.decode(tokenizer.encode(second_half).last(truncation_length)).length
+ tail = second_half[(second_half.length - tail_length)..]
+ item[:text] = "#{head} #{tail}"
item
end
@@ -147,10 +119,6 @@ module DiscourseAi
# @param items { Array<Hash> } - Content to summarize. Structure will be: { poster: who wrote the content, id: a way to order content, text: content }
# @param user { User } - User object used for auditing usage.
# @param &on_partial_blk { Block - Optional } - The passed block will get called with the LLM partial response.
- # Note: The block is only called with results of the final summary, not intermediate summaries.
- #
- # The summarization algorithm.
- # It will summarize as much content summarize given the model's context window. If will prioriotize newer content in case it doesn't fit.
#
# @returns { String } - Resulting summary.
def fold(items, user, &on_partial_blk)
@@ -158,7 +126,7 @@ module DiscourseAi
tokens_left = available_tokens
content_in_window = []
- items.each_with_index do |item, idx|
+ items.each do |item|
as_text = "(#{item[:id]} #{item[:poster]} said: #{item[:text]} "
if tokenizer.below_limit?(
diff --git a/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rb b/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rb
index ab4f9bfa103..d12f9b5feba 100644
--- a/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/summarization/fold_content_spec.rb
@@ -237,7 +237,7 @@ RSpec.describe DiscourseAi::Summarization::FoldContent do
expect(regional_summarizer.existing_summary).to eq(existing_gist)
end
- it "finds and deletes a complete summary stored under an equivalent regional locale" do
+ it "finds a complete summary stored under an equivalent regional locale" do
existing_summary =
Fabricate(
:ai_summary,
@@ -249,9 +249,6 @@ RSpec.describe DiscourseAi::Summarization::FoldContent do
regional_summarizer = DiscourseAi::Summarization.topic_summary(topic, locale: "pt_BR")
expect(regional_summarizer.existing_summary).to eq(existing_summary)
-
- regional_summarizer.delete_cached_summaries!
- expect(AiSummary.find_by(id: existing_summary.id)).to be_nil
end
context "when a summary already exists" do
@@ -289,24 +286,30 @@ RSpec.describe DiscourseAi::Summarization::FoldContent do
end
describe "#truncate" do
- it "preserves grapheme clusters for multi-codepoint emoji sequences" do
- # Starts with scales emoji (⚖️ = U+2696 + U+FE0F) so we can catch any split between code points.
- sample_text = "⚖️🧩"
+ it "leaves content within the token limit untouched" do
+ text = "用 [keyd](https://man.archlinux.org/man/extra/keyd/keyd.1.en) 改键"
+
+ expect(summarizer.truncate({ text: text.dup })[:text]).to eq(text)
+ end
- item = summarizer.truncate({ text: sample_text.dup })
+ it "doesn't let the tokenizer rewrite content within the token limit" do
+ llm_model.update!(tokenizer: "DiscourseAi::Tokenizer::AnthropicTokenizer")
+ text = "改键,很好用。" * 100
- expect(item[:text]).to start_with("⚖️ ")
- expect(item[:text]).to include("🧩")
- expect(item[:text]).not_to start_with("⚖ ️")
+ expect(summarizer.truncate({ text: text.dup })[:text]).to eq(text)
end
- it "keeps the second half of the text in the original order" do
- sample_text = "abcdefgh"
+ it "keeps the start and the end of longer content" do
+ item = summarizer.truncate({ text: "start #{"a " * 1500}#{"b " * 1500}end" })
+
+ expect(item[:text]).to start_with("start a")
+ expect(item[:text]).to end_with("b end")
+ end
- item = summarizer.truncate({ text: sample_text.dup })
+ it "preserves grapheme clusters at the split point" do
+ item = summarizer.truncate({ text: "#{"🧩" * 600}⚖️#{"a" * 600}" })
- expect(item[:text]).to include("efgh")
- expect(item[:text]).not_to include("hgfe")
+ expect(item[:text]).to include(" ⚖️a")
end
end
end