Skip to content

A platform for community discussion. Free, open, simple.

FIX: Preserve boundaries and token budgets in AI translation (#43605)

Régis Hanol58a5c119bb79
diff --git a/plugins/discourse-ai/lib/translation/base_translator.rb b/plugins/discourse-ai/lib/translation/base_translator.rb
index 01367a49d20..2bb9426ed0c 100644
--- a/plugins/discourse-ai/lib/translation/base_translator.rb
+++ b/plugins/discourse-ai/lib/translation/base_translator.rb
@@ -3,6 +3,10 @@
module DiscourseAi
module Translation
class BaseTranslator
+ DEFAULT_OUTPUT_TOKEN_BUDGET = 4096
+ TRANSLATION_EXPANSION_FACTOR = 2
+ PROMPT_TOKEN_RESERVE = 256
+
def initialize(
text:,
target_locale:,
@@ -33,11 +37,18 @@ module DiscourseAi
return nil if model.blank?
bot = DiscourseAi::Agents::Bot.as(translation_user, agent:, model:)
+ tokenizer = model.tokenizer_class
+ payload_overhead = tokenizer.size(formatted_content(""))
+ output_tokens = output_token_budget(model, agent, payload_overhead)
+ chunk_size = output_tokens / TRANSLATION_EXPANSION_FACTOR
+ chunks =
+ ContentSplitter.split(content: @text, chunk_size:) do |text|
+ [tokenizer.size(text), tokenizer.size(formatted_content(text)) - payload_overhead].max
+ end
translated =
- ContentSplitter
- .split(content: @text, chunk_size: model.max_output_tokens)
- .map { |text| get_translation(text:, bot:, translation_user:, model:) }
+ chunks
+ .map { |text| get_translation(text:, bot:, translation_user:, model:, output_tokens:) }
.join("")
strip_control_characters(translated)
@@ -45,6 +56,22 @@ module DiscourseAi
private
+ def output_token_budget(model, agent, payload_overhead)
+ prompt_tokens =
+ [
+ agent.system_prompt,
+ JSON.generate(agent.response_format),
+ *Array(agent.examples).flatten,
+ ].sum { |text| model.tokenizer_class.size(text) }
+ available_tokens =
+ model.max_prompt_tokens - prompt_tokens - payload_overhead - PROMPT_TOKEN_RESERVE
+
+ [
+ model.max_output_tokens || DEFAULT_OUTPUT_TOKEN_BUDGET,
+ available_tokens * TRANSLATION_EXPANSION_FACTOR / (TRANSLATION_EXPANSION_FACTOR + 1),
+ ].min
+ end
+
def formatted_content(content)
payload = { content:, target_locale: @target_locale }
payload[:content_description] = @content_description if @content_description.present?
@@ -61,7 +88,7 @@ module DiscourseAi
text.gsub(/[\u0000-\u0008\u000B-\u001F\u007F\u0080-\u009F]/, "")
end
- def get_translation(text:, bot:, translation_user:, model:)
+ def get_translation(text:, bot:, translation_user:, model:, output_tokens:)
context =
DiscourseAi::Agents::BotContext.new(
user: translation_user,
@@ -71,7 +98,7 @@ module DiscourseAi
topic: @topic,
post: @post,
)
- llm_args = { max_tokens: model.max_output_tokens }
+ llm_args = { max_tokens: model.max_output_tokens && output_tokens }
structured_output = nil
result = +""
diff --git a/plugins/discourse-ai/lib/translation/content_splitter.rb b/plugins/discourse-ai/lib/translation/content_splitter.rb
index e2d2a069818..2f07b26dc10 100644
--- a/plugins/discourse-ai/lib/translation/content_splitter.rb
+++ b/plugins/discourse-ai/lib/translation/content_splitter.rb
@@ -23,20 +23,26 @@ module DiscourseAi
/\s+/, # any whitespace
].freeze
- def self.split(content:, chunk_size: DEFAULT_CHUNK_SIZE)
+ def self.split(content:, chunk_size: DEFAULT_CHUNK_SIZE, &measure)
return [] if content.nil?
return [""] if content.empty?
- chunk_size ||= DEFAULT_CHUNK_SIZE
- return [content] if content.length <= chunk_size
+ measure ||= :length.to_proc
chunks = []
- remaining = content.dup
+ offset = 0
+
+ while offset < content.length
+ chunk = content[offset..]
+ loop do
+ length = fitting_length(chunk, chunk_size, &measure)
+ break if length == chunk.length
+
+ chunk = extract_mixed_chunk(chunk, size: length)
+ break if measure.call(chunk) <= chunk_size
+ end
- while remaining.present?
- chunk = extract_mixed_chunk(remaining, size: chunk_size)
- break if chunk.empty?
chunks << chunk
- remaining = remaining[chunk.length..-1]
+ offset += chunk.length
end
chunks
@@ -44,65 +50,77 @@ module DiscourseAi
private
- def self.extract_mixed_chunk(text, size:)
- return text if text.length <= size
+ def self.fitting_length(text, size, &measure)
+ return text.length if measure.call(text) <= size
+
+ length = (1..text.length).bsearch { |n| measure.call(text[0...n]) > size } - 1
+ raise ArgumentError, "Chunk size is too small to fit content" if length == 0
+
+ length
+ end
- # try each splitting strategy in order
+ def self.extract_mixed_chunk(text, size:)
split_point =
- [
- -> { find_nearest_html_end_index(text, size) },
- -> { find_nearest_bbcode_end_index(text, size) },
- -> { find_text_boundary(text, size) },
- -> { size },
- ].lazy.map(&:call).compact.find { |pos| pos <= size }
+ find_nearest_bbcode_end_index(text, size) || find_nearest_html_end_index(text, size) ||
+ find_text_boundary(text, size) || size
+ split_point += 1 while split_point < size && text[split_point].match?(/\s/)
text[0...split_point]
end
def self.find_nearest_html_end_index(text, target_pos)
- return nil if !text.include?("<")
-
- begin
- doc = Nokogiri::HTML5.fragment(text)
- max_length_within_target = 0
+ return if !text.include?("<")
- doc.children.each do |node|
- html = node.to_html
- end_pos = max_length_within_target + html.length
- return max_length_within_target if max_length_within_target > 0 && end_pos > target_pos
- max_length_within_target = end_pos
- end
- nil
- rescue Nokogiri::SyntaxError
- nil
- end
+ handler = HtmlBoundaries.new(text, target_pos)
+ Nokogiri::HTML4::SAX::Parser
+ .new(handler, Encoding::UTF_8)
+ .parse_memory(text) { |context| handler.context = context }
+ handler.element_end || handler.text_end
end
def self.find_nearest_bbcode_end_index(text, target_pos)
- max_length_within_target = 0
- BBCODE_PATTERNS.each do |pattern|
- text.scan(pattern) do |_|
- match = $~
- tag_end = match.end(0)
- return max_length_within_target if max_length_within_target > 0 && tag_end > target_pos
- max_length_within_target = tag_end
+ BBCODE_PATTERNS
+ .flat_map do |pattern|
+ text.to_enum(:scan, pattern).flat_map { Regexp.last_match.offset(0) }
end
- end
+ .select { |position| position.positive? && position <= target_pos }
+ .max
+ end
+ def self.find_text_boundary(text, target_pos)
+ head = text[0...target_pos]
+ TEXT_BOUNDARIES.each { |pattern| return Regexp.last_match.end(0) if head.rindex(pattern) }
nil
end
- def self.find_text_boundary(text, target_pos)
- search_text = text
+ class HtmlBoundaries < Nokogiri::XML::SAX::Document
+ attr_accessor :context
+ attr_reader :element_end, :text_end
- TEXT_BOUNDARIES.each do |pattern|
- if pos = search_text.rindex(pattern, target_pos)
- # Include all trailing whitespace
- pos += 1 while pos < search_text.length && search_text[pos].match?(/\s/)
- return pos
- end
+ def initialize(text, target_pos)
+ @target_pos = target_pos
+ @line_offsets = [0]
+ text.each_line { |line| @line_offsets << @line_offsets.last + line.length }
+ end
+
+ def end_element(_name)
+ @element_end = source_position || @element_end
+ end
+
+ alias_method :comment, :end_element
+
+ def characters(_text)
+ @text_end = source_position || @text_end
+ end
+
+ alias_method :cdata_block, :characters
+
+ private
+
+ def source_position
+ position = @line_offsets[context.line - 1] + context.column - 1
+ position if position.positive? && position <= @target_pos
end
- nil
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
index 517596e51e1..644dfa01b66 100644
--- a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
+++ b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
@@ -20,98 +20,108 @@ describe DiscourseAi::Translation::BaseTranslator do
fab!(:post)
fab!(:topic) { post.topic }
- it "creates the correct prompt" do
- post_translator =
- DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:, post:)
- allow(DiscourseAi::Completions::Prompt).to receive(:new).with(
- agent.system_prompt,
- messages: array_including({ type: :user, content: a_string_including(text) }),
- post_id: post.id,
- topic_id: post.topic_id,
- ).and_call_original
-
- DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do
- post_translator.translate
- end
- end
-
- it "creates BotContext with the correct parameters and calls bot.reply with model max_output_tokens" do
- post_translator =
+ it "sends unescaped content, post context, and the output limit to the model" do
+ text = "List<string> and ?a=1&b=2"
+ llm_model = Fabricate(:fake_model, max_output_tokens: 256)
+ translator =
DiscourseAi::Translation::PostRawTranslator.new(
text:,
target_locale:,
content_description:,
post:,
topic:,
+ llm_model:,
)
- expected_content = {
- content: text,
- target_locale: target_locale,
- content_description:,
- }.to_json
-
- bot_context = instance_double(DiscourseAi::Agents::BotContext)
- allow(DiscourseAi::Agents::BotContext).to receive(:new).and_return(bot_context)
-
- mock_bot = instance_double(DiscourseAi::Agents::Bot)
- allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(mock_bot)
- allow(mock_bot).to receive(:reply).and_yield(llm_response, nil, nil)
-
- post_translator.translate
-
- expect(DiscourseAi::Agents::BotContext).to have_received(:new).with(
- user: an_instance_of(User),
- skip_show_thinking: true,
- feature_name: "translation",
- messages: [{ type: :user, content: expected_content }],
- topic: topic,
- post: post,
- )
-
- expect(DiscourseAi::Agents::Bot).to have_received(:as)
- expect(mock_bot).to have_received(:reply).with(
- bot_context,
- llm_args: {
- max_tokens: LlmModel.last.max_output_tokens,
- },
- )
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [llm_response],
+ ) do |_, _, prompts, options|
+ translator.translate
+ prompt = prompts.last
+
+ expect(prompt.system_message_text).to eq(agent.system_prompt)
+ expect(prompt.messages.last).to eq(
+ type: :user,
+ content: JSON.generate(content: text, target_locale:, content_description:),
+ )
+ expect([prompt.post_id, prompt.topic_id]).to eq([post.id, topic.id])
+ expect(options.last[:max_tokens]).to eq(llm_model.max_output_tokens)
+ end
end
- it "returns the translation from the llm's response" do
- DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do
- expect(
- DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:).translate,
- ).to eq "hur dur hur dur!"
+ it "fits a post by token count without charging descriptive context to the output budget" do
+ llm_model = Fabricate(:fake_model, max_output_tokens: 256)
+ source = "hello world " * 30
+ translator =
+ DiscourseAi::Translation::PostRawTranslator.new(
+ text: source,
+ target_locale:,
+ content_description: "Background context " * 100,
+ llm_model:,
+ )
+
+ DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do |_, _, prompts|
+ expect(translator.translate).to eq(llm_response)
+ expect(JSON.parse(prompts.last.messages.last[:content])["content"]).to eq(source)
end
end
- it "sends the content to the model without HTML entity escaping" do
- html_text = "List<string> and ?a=1&b=2"
- post_translator =
- DiscourseAi::Translation::PostRawTranslator.new(text: html_text, target_locale:, post:)
-
- bot_context = instance_double(DiscourseAi::Agents::BotContext)
- allow(DiscourseAi::Agents::BotContext).to receive(:new).and_return(bot_context)
- mock_bot = instance_double(DiscourseAi::Agents::Bot)
- allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(mock_bot)
- allow(mock_bot).to receive(:reply).and_yield(llm_response, nil, nil)
-
- post_translator.translate
-
- expect(DiscourseAi::Agents::BotContext).to have_received(:new).with(
- user: an_instance_of(User),
- skip_show_thinking: true,
- feature_name: "translation",
- messages: [
- {
- type: :user,
- content: "{\"content\":\"List<string> and ?a=1&b=2\",\"target_locale\":\"de\"}",
- },
- ],
- topic: nil,
- post: post,
- )
+ it "reserves output headroom and preserves all source text across requests" do
+ llm_model = Fabricate(:fake_model, max_output_tokens: 128)
+ source = "猫と犬が好きです。" * 40
+ translator =
+ DiscourseAi::Translation::PostRawTranslator.new(text: source, target_locale:, llm_model:)
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ Array.new(40, "translated"),
+ ) do |_, _, prompts|
+ result = translator.translate
+ chunks = prompts.map { |prompt| JSON.parse(prompt.messages.last[:content])["content"] }
+
+ expect(result).to eq("translated" * chunks.size)
+ expect(chunks.join).to eq(source)
+ expect(chunks.map { |chunk| llm_model.tokenizer_class.size(chunk) }).to all(be <= 64)
+ end
+ end
+
+ it "accounts for the system prompt, examples, and content description" do
+ llm_model = Fabricate(:fake_model, max_output_tokens: 8192, max_prompt_tokens: 4096)
+ source = "hello world " * 3000
+ translator =
+ DiscourseAi::Translation::PostRawTranslator.new(
+ text: source,
+ target_locale:,
+ content_description: "A long description " * 100,
+ llm_model:,
+ )
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ Array.new(40, "translated"),
+ ) do |_, _, prompts, options|
+ translator.translate
+ chunks = prompts.map { |prompt| JSON.parse(prompt.messages.last[:content])["content"] }
+
+ expect(chunks.join).to eq(source)
+ request_tokens =
+ prompts
+ .zip(options)
+ .map do |prompt, option|
+ prompt.messages.sum { |message| llm_model.tokenizer_class.size(message[:content]) } +
+ option[:max_tokens]
+ end
+ expect(request_tokens).to all(be <= llm_model.max_prompt_tokens)
+ end
+ end
+
+ it "leaves the output limit to the provider when the model has none" do
+ llm_model = Fabricate(:fake_model, max_output_tokens: nil, max_prompt_tokens: 8192)
+ translator =
+ DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:, llm_model:)
+
+ DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do |_, _, _, options|
+ expect(translator.translate).to eq(llm_response)
+ expect(options.last[:max_tokens]).to be_nil
+ end
end
it "strips control characters from the model response but keeps newlines" do
diff --git a/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rb b/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rb
index db4d956417d..ff3fbf97a5e 100644
--- a/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rb
+++ b/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rb
@@ -36,6 +36,60 @@ describe DiscourseAi::Translation::ContentSplitter do
)
end
+ it "preserves source boundaries when HTML serialization changes the length" do
+ ["<p>&#65;</p>", "<p title='a < b > c'>hello</p>"].each do |html|
+ content = "#{html}<p>world</p>"
+
+ expect(described_class.split(content:, chunk_size: html.length + 3)).to eq(
+ [html, "<p>world</p>"],
+ )
+ end
+ end
+
+ it "preserves source offsets across Unicode characters and CRLF line endings" do
+ prefix = "<p>猫&#65;</p>\r\n<p>é</p>\r\n"
+
+ expect(described_class.split(content: "#{prefix}<p>end</p>", chunk_size: 30)).to eq(
+ [prefix, "<p>end</p>"],
+ )
+ end
+
+ it "keeps entities and self-closing tags intact" do
+ content = "猫 &#x1F408;<br/><p>hello</p>"
+
+ expect(described_class.split(content:, chunk_size: 12)).to eq(
+ ["猫 &#x1F408;", "<br/>", "<p>hello</p>"],
+ )
+ end
+
+ it "rejects a budget that cannot fit one measured character" do
+ expect { described_class.split(content: "猫", chunk_size: 1, &:bytesize) }.to raise_error(
+ ArgumentError,
+ "Chunk size is too small to fit content",
+ )
+ end
+
+ it "keeps mixed markup blocks intact when they fit" do
+ quote = "[quote]hello world[/quote]"
+ html = "<p>next</p>"
+ content = "intro #{quote}#{html}"
+
+ expect(described_class.split(content:, chunk_size: quote.length)).to eq(["intro ", quote, html])
+ end
+
+ it "makes progress when a sentence boundary starts at the beginning" do
+ content = ". #{"a" * 30}"
+
+ expect(described_class.split(content:, chunk_size: 10).join).to eq(content)
+ end
+
+ it "uses the supplied size measurement without splitting Unicode characters" do
+ content = "猫 犬 鳥 魚"
+ chunks = described_class.split(content:, chunk_size: 7, &:bytesize)
+
+ expect(chunks).to eq(["猫 ", "犬 ", "鳥 魚"])
+ end
+
it "preserves BBCode tags" do
content = "[quote]hello[/quote][details]world[/details]"
expect(described_class.split(content:, chunk_size: 25)).to eq(
@@ -79,9 +133,9 @@ describe DiscourseAi::Translation::ContentSplitter do
end
it "keeps code blocks intact" do
- content = "Text\n```\ncode block\nhere\n```\nmore text"
+ content = "Text\n```\n<div>\nhere\n```\nmore text"
expect(described_class.split(content:, chunk_size: 30)).to eq(
- ["Text\n```\ncode block\nhere\n```\n", "more text"],
+ ["Text\n```\n<div>\nhere\n```\n", "more text"],
)
end