A platform for community discussion. Free, open, simple.
FIX: Preserve boundaries and token budgets in AI translation (#43605)
Régis Hanol58a5c119bb79
diff --git a/plugins/discourse-ai/lib/translation/base_translator.rb b/plugins/discourse-ai/lib/translation/base_translator.rbindex 01367a49d20..2bb9426ed0c 100644--- a/plugins/discourse-ai/lib/translation/base_translator.rb+++ b/plugins/discourse-ai/lib/translation/base_translator.rb@@ -3,6 +3,10 @@module DiscourseAimodule Translationclass BaseTranslator+ DEFAULT_OUTPUT_TOKEN_BUDGET = 4096+ TRANSLATION_EXPANSION_FACTOR = 2+ PROMPT_TOKEN_RESERVE = 256+def initialize(text:,target_locale:,@@ -33,11 +37,18 @@ module DiscourseAireturn nil if model.blank?bot = DiscourseAi::Agents::Bot.as(translation_user, agent:, model:)+ tokenizer = model.tokenizer_class+ payload_overhead = tokenizer.size(formatted_content(""))+ output_tokens = output_token_budget(model, agent, payload_overhead)+ chunk_size = output_tokens / TRANSLATION_EXPANSION_FACTOR+ chunks =+ ContentSplitter.split(content: @text, chunk_size:) do |text|+ [tokenizer.size(text), tokenizer.size(formatted_content(text)) - payload_overhead].max+ endtranslated =- ContentSplitter- .split(content: @text, chunk_size: model.max_output_tokens)- .map { |text| get_translation(text:, bot:, translation_user:, model:) }+ chunks+ .map { |text| get_translation(text:, bot:, translation_user:, model:, output_tokens:) }.join("")strip_control_characters(translated)@@ -45,6 +56,22 @@ module DiscourseAiprivate+ def output_token_budget(model, agent, payload_overhead)+ prompt_tokens =+ [+ agent.system_prompt,+ JSON.generate(agent.response_format),+ *Array(agent.examples).flatten,+ ].sum { |text| model.tokenizer_class.size(text) }+ available_tokens =+ model.max_prompt_tokens - prompt_tokens - payload_overhead - PROMPT_TOKEN_RESERVE++ [+ model.max_output_tokens || DEFAULT_OUTPUT_TOKEN_BUDGET,+ available_tokens * TRANSLATION_EXPANSION_FACTOR / (TRANSLATION_EXPANSION_FACTOR + 1),+ ].min+ end+def formatted_content(content)payload = { content:, target_locale: @target_locale }payload[:content_description] = @content_description if @content_description.present?@@ -61,7 +88,7 @@ module DiscourseAitext.gsub(/[\u0000-\u0008\u000B-\u001F\u007F\u0080-\u009F]/, "")end- def get_translation(text:, bot:, translation_user:, model:)+ def get_translation(text:, bot:, translation_user:, model:, output_tokens:)context =DiscourseAi::Agents::BotContext.new(user: translation_user,@@ -71,7 +98,7 @@ module DiscourseAitopic: @topic,post: @post,)- llm_args = { max_tokens: model.max_output_tokens }+ llm_args = { max_tokens: model.max_output_tokens && output_tokens }structured_output = nilresult = +""diff --git a/plugins/discourse-ai/lib/translation/content_splitter.rb b/plugins/discourse-ai/lib/translation/content_splitter.rbindex e2d2a069818..2f07b26dc10 100644--- a/plugins/discourse-ai/lib/translation/content_splitter.rb+++ b/plugins/discourse-ai/lib/translation/content_splitter.rb@@ -23,20 +23,26 @@ module DiscourseAi/\s+/, # any whitespace].freeze- def self.split(content:, chunk_size: DEFAULT_CHUNK_SIZE)+ def self.split(content:, chunk_size: DEFAULT_CHUNK_SIZE, &measure)return [] if content.nil?return [""] if content.empty?- chunk_size ||= DEFAULT_CHUNK_SIZE- return [content] if content.length <= chunk_size+ measure ||= :length.to_procchunks = []- remaining = content.dup+ offset = 0++ while offset < content.length+ chunk = content[offset..]+ loop do+ length = fitting_length(chunk, chunk_size, &measure)+ break if length == chunk.length++ chunk = extract_mixed_chunk(chunk, size: length)+ break if measure.call(chunk) <= chunk_size+ end- while remaining.present?- chunk = extract_mixed_chunk(remaining, size: chunk_size)- break if chunk.empty?chunks << chunk- remaining = remaining[chunk.length..-1]+ offset += chunk.lengthendchunks@@ -44,65 +50,77 @@ module DiscourseAiprivate- def self.extract_mixed_chunk(text, size:)- return text if text.length <= size+ def self.fitting_length(text, size, &measure)+ return text.length if measure.call(text) <= size++ length = (1..text.length).bsearch { |n| measure.call(text[0...n]) > size } - 1+ raise ArgumentError, "Chunk size is too small to fit content" if length == 0++ length+ end- # try each splitting strategy in order+ def self.extract_mixed_chunk(text, size:)split_point =- [- -> { find_nearest_html_end_index(text, size) },- -> { find_nearest_bbcode_end_index(text, size) },- -> { find_text_boundary(text, size) },- -> { size },- ].lazy.map(&:call).compact.find { |pos| pos <= size }+ find_nearest_bbcode_end_index(text, size) || find_nearest_html_end_index(text, size) ||+ find_text_boundary(text, size) || size+ split_point += 1 while split_point < size && text[split_point].match?(/\s/)text[0...split_point]enddef self.find_nearest_html_end_index(text, target_pos)- return nil if !text.include?("<")-- begin- doc = Nokogiri::HTML5.fragment(text)- max_length_within_target = 0+ return if !text.include?("<")- doc.children.each do |node|- html = node.to_html- end_pos = max_length_within_target + html.length- return max_length_within_target if max_length_within_target > 0 && end_pos > target_pos- max_length_within_target = end_pos- end- nil- rescue Nokogiri::SyntaxError- nil- end+ handler = HtmlBoundaries.new(text, target_pos)+ Nokogiri::HTML4::SAX::Parser+ .new(handler, Encoding::UTF_8)+ .parse_memory(text) { |context| handler.context = context }+ handler.element_end || handler.text_endenddef self.find_nearest_bbcode_end_index(text, target_pos)- max_length_within_target = 0- BBCODE_PATTERNS.each do |pattern|- text.scan(pattern) do |_|- match = $~- tag_end = match.end(0)- return max_length_within_target if max_length_within_target > 0 && tag_end > target_pos- max_length_within_target = tag_end+ BBCODE_PATTERNS+ .flat_map do |pattern|+ text.to_enum(:scan, pattern).flat_map { Regexp.last_match.offset(0) }end- end+ .select { |position| position.positive? && position <= target_pos }+ .max+ end+ def self.find_text_boundary(text, target_pos)+ head = text[0...target_pos]+ TEXT_BOUNDARIES.each { |pattern| return Regexp.last_match.end(0) if head.rindex(pattern) }nilend- def self.find_text_boundary(text, target_pos)- search_text = text+ class HtmlBoundaries < Nokogiri::XML::SAX::Document+ attr_accessor :context+ attr_reader :element_end, :text_end- TEXT_BOUNDARIES.each do |pattern|- if pos = search_text.rindex(pattern, target_pos)- # Include all trailing whitespace- pos += 1 while pos < search_text.length && search_text[pos].match?(/\s/)- return pos- end+ def initialize(text, target_pos)+ @target_pos = target_pos+ @line_offsets = [0]+ text.each_line { |line| @line_offsets << @line_offsets.last + line.length }+ end++ def end_element(_name)+ @element_end = source_position || @element_end+ end++ alias_method :comment, :end_element++ def characters(_text)+ @text_end = source_position || @text_end+ end++ alias_method :cdata_block, :characters++ private++ def source_position+ position = @line_offsets[context.line - 1] + context.column - 1+ position if position.positive? && position <= @target_posend- nilendendenddiff --git a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rbindex 517596e51e1..644dfa01b66 100644--- a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb@@ -20,98 +20,108 @@ describe DiscourseAi::Translation::BaseTranslator dofab!(:post)fab!(:topic) { post.topic }- it "creates the correct prompt" do- post_translator =- DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:, post:)- allow(DiscourseAi::Completions::Prompt).to receive(:new).with(- agent.system_prompt,- messages: array_including({ type: :user, content: a_string_including(text) }),- post_id: post.id,- topic_id: post.topic_id,- ).and_call_original-- DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do- post_translator.translate- end- end-- it "creates BotContext with the correct parameters and calls bot.reply with model max_output_tokens" do- post_translator =+ it "sends unescaped content, post context, and the output limit to the model" do+ text = "List<string> and ?a=1&b=2"+ llm_model = Fabricate(:fake_model, max_output_tokens: 256)+ translator =DiscourseAi::Translation::PostRawTranslator.new(text:,target_locale:,content_description:,post:,topic:,+ llm_model:,)- expected_content = {- content: text,- target_locale: target_locale,- content_description:,- }.to_json-- bot_context = instance_double(DiscourseAi::Agents::BotContext)- allow(DiscourseAi::Agents::BotContext).to receive(:new).and_return(bot_context)-- mock_bot = instance_double(DiscourseAi::Agents::Bot)- allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(mock_bot)- allow(mock_bot).to receive(:reply).and_yield(llm_response, nil, nil)-- post_translator.translate-- expect(DiscourseAi::Agents::BotContext).to have_received(:new).with(- user: an_instance_of(User),- skip_show_thinking: true,- feature_name: "translation",- messages: [{ type: :user, content: expected_content }],- topic: topic,- post: post,- )-- expect(DiscourseAi::Agents::Bot).to have_received(:as)- expect(mock_bot).to have_received(:reply).with(- bot_context,- llm_args: {- max_tokens: LlmModel.last.max_output_tokens,- },- )+ DiscourseAi::Completions::Llm.with_prepared_responses(+ [llm_response],+ ) do |_, _, prompts, options|+ translator.translate+ prompt = prompts.last++ expect(prompt.system_message_text).to eq(agent.system_prompt)+ expect(prompt.messages.last).to eq(+ type: :user,+ content: JSON.generate(content: text, target_locale:, content_description:),+ )+ expect([prompt.post_id, prompt.topic_id]).to eq([post.id, topic.id])+ expect(options.last[:max_tokens]).to eq(llm_model.max_output_tokens)+ endend- it "returns the translation from the llm's response" do- DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do- expect(- DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:).translate,- ).to eq "hur dur hur dur!"+ it "fits a post by token count without charging descriptive context to the output budget" do+ llm_model = Fabricate(:fake_model, max_output_tokens: 256)+ source = "hello world " * 30+ translator =+ DiscourseAi::Translation::PostRawTranslator.new(+ text: source,+ target_locale:,+ content_description: "Background context " * 100,+ llm_model:,+ )++ DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do |_, _, prompts|+ expect(translator.translate).to eq(llm_response)+ expect(JSON.parse(prompts.last.messages.last[:content])["content"]).to eq(source)endend- it "sends the content to the model without HTML entity escaping" do- html_text = "List<string> and ?a=1&b=2"- post_translator =- DiscourseAi::Translation::PostRawTranslator.new(text: html_text, target_locale:, post:)-- bot_context = instance_double(DiscourseAi::Agents::BotContext)- allow(DiscourseAi::Agents::BotContext).to receive(:new).and_return(bot_context)- mock_bot = instance_double(DiscourseAi::Agents::Bot)- allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(mock_bot)- allow(mock_bot).to receive(:reply).and_yield(llm_response, nil, nil)-- post_translator.translate-- expect(DiscourseAi::Agents::BotContext).to have_received(:new).with(- user: an_instance_of(User),- skip_show_thinking: true,- feature_name: "translation",- messages: [- {- type: :user,- content: "{\"content\":\"List<string> and ?a=1&b=2\",\"target_locale\":\"de\"}",- },- ],- topic: nil,- post: post,- )+ it "reserves output headroom and preserves all source text across requests" do+ llm_model = Fabricate(:fake_model, max_output_tokens: 128)+ source = "猫と犬が好きです。" * 40+ translator =+ DiscourseAi::Translation::PostRawTranslator.new(text: source, target_locale:, llm_model:)++ DiscourseAi::Completions::Llm.with_prepared_responses(+ Array.new(40, "translated"),+ ) do |_, _, prompts|+ result = translator.translate+ chunks = prompts.map { |prompt| JSON.parse(prompt.messages.last[:content])["content"] }++ expect(result).to eq("translated" * chunks.size)+ expect(chunks.join).to eq(source)+ expect(chunks.map { |chunk| llm_model.tokenizer_class.size(chunk) }).to all(be <= 64)+ end+ end++ it "accounts for the system prompt, examples, and content description" do+ llm_model = Fabricate(:fake_model, max_output_tokens: 8192, max_prompt_tokens: 4096)+ source = "hello world " * 3000+ translator =+ DiscourseAi::Translation::PostRawTranslator.new(+ text: source,+ target_locale:,+ content_description: "A long description " * 100,+ llm_model:,+ )++ DiscourseAi::Completions::Llm.with_prepared_responses(+ Array.new(40, "translated"),+ ) do |_, _, prompts, options|+ translator.translate+ chunks = prompts.map { |prompt| JSON.parse(prompt.messages.last[:content])["content"] }++ expect(chunks.join).to eq(source)+ request_tokens =+ prompts+ .zip(options)+ .map do |prompt, option|+ prompt.messages.sum { |message| llm_model.tokenizer_class.size(message[:content]) } ++ option[:max_tokens]+ end+ expect(request_tokens).to all(be <= llm_model.max_prompt_tokens)+ end+ end++ it "leaves the output limit to the provider when the model has none" do+ llm_model = Fabricate(:fake_model, max_output_tokens: nil, max_prompt_tokens: 8192)+ translator =+ DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:, llm_model:)++ DiscourseAi::Completions::Llm.with_prepared_responses([llm_response]) do |_, _, _, options|+ expect(translator.translate).to eq(llm_response)+ expect(options.last[:max_tokens]).to be_nil+ endendit "strips control characters from the model response but keeps newlines" dodiff --git a/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rb b/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rbindex db4d956417d..ff3fbf97a5e 100644--- a/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/content_splitter_spec.rb@@ -36,6 +36,60 @@ describe DiscourseAi::Translation::ContentSplitter do)end+ it "preserves source boundaries when HTML serialization changes the length" do+ ["<p>A</p>", "<p title='a < b > c'>hello</p>"].each do |html|+ content = "#{html}<p>world</p>"++ expect(described_class.split(content:, chunk_size: html.length + 3)).to eq(+ [html, "<p>world</p>"],+ )+ end+ end++ it "preserves source offsets across Unicode characters and CRLF line endings" do+ prefix = "<p>猫A</p>\r\n<p>é</p>\r\n"++ expect(described_class.split(content: "#{prefix}<p>end</p>", chunk_size: 30)).to eq(+ [prefix, "<p>end</p>"],+ )+ end++ it "keeps entities and self-closing tags intact" do+ content = "猫 🐈<br/><p>hello</p>"++ expect(described_class.split(content:, chunk_size: 12)).to eq(+ ["猫 🐈", "<br/>", "<p>hello</p>"],+ )+ end++ it "rejects a budget that cannot fit one measured character" do+ expect { described_class.split(content: "猫", chunk_size: 1, &:bytesize) }.to raise_error(+ ArgumentError,+ "Chunk size is too small to fit content",+ )+ end++ it "keeps mixed markup blocks intact when they fit" do+ quote = "[quote]hello world[/quote]"+ html = "<p>next</p>"+ content = "intro #{quote}#{html}"++ expect(described_class.split(content:, chunk_size: quote.length)).to eq(["intro ", quote, html])+ end++ it "makes progress when a sentence boundary starts at the beginning" do+ content = ". #{"a" * 30}"++ expect(described_class.split(content:, chunk_size: 10).join).to eq(content)+ end++ it "uses the supplied size measurement without splitting Unicode characters" do+ content = "猫 犬 鳥 魚"+ chunks = described_class.split(content:, chunk_size: 7, &:bytesize)++ expect(chunks).to eq(["猫 ", "犬 ", "鳥 魚"])+ end+it "preserves BBCode tags" docontent = "[quote]hello[/quote][details]world[/details]"expect(described_class.split(content:, chunk_size: 25)).to eq(@@ -79,9 +133,9 @@ describe DiscourseAi::Translation::ContentSplitter doendit "keeps code blocks intact" do- content = "Text\n```\ncode block\nhere\n```\nmore text"+ content = "Text\n```\n<div>\nhere\n```\nmore text"expect(described_class.split(content:, chunk_size: 30)).to eq(- ["Text\n```\ncode block\nhere\n```\n", "more text"],+ ["Text\n```\n<div>\nhere\n```\n", "more text"],)end