A platform for community discussion. Free, open, simple.
FIX: Control characters injected into AI translations (#41743)
Rafael dos Santos Silvac0334fa7a32f
diff --git a/plugins/discourse-ai/lib/translation/base_translator.rb b/plugins/discourse-ai/lib/translation/base_translator.rbindex 901c929a7a4..43ba682bc8d 100644--- a/plugins/discourse-ai/lib/translation/base_translator.rb+++ b/plugins/discourse-ai/lib/translation/base_translator.rb@@ -26,16 +26,28 @@ module DiscourseAibot = DiscourseAi::Agents::Bot.as(translation_user, agent:, model:)- ContentSplitter- .split(content: @text, chunk_size: model.max_output_tokens)- .map { |text| get_translation(text:, bot:, translation_user:, model:) }- .join("")+ translated =+ ContentSplitter+ .split(content: @text, chunk_size: model.max_output_tokens)+ .map { |text| get_translation(text:, bot:, translation_user:, model:) }+ .join("")++ strip_control_characters(translated)endprivatedef formatted_content(content)- { content:, target_locale: @target_locale }.to_json+ # JSON.generate over to_json: ActiveSupport HTML-escapes <, >, and & into+ # \uXXXX sequences, which models can mis-copy into control characters+ JSON.generate({ content:, target_locale: @target_locale })+ end++ # control characters are never valid in a translation, but models+ # occasionally emit them by mangling unicode escapes (e.g. \u003c+ # coming back as \u001c)+ def strip_control_characters(text)+ text.gsub(/[\u0000-\u0008\u000B-\u001F\u007F\u0080-\u009F]/, "")enddef get_translation(text:, bot:, translation_user:, model:)diff --git a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rbindex 21ce0e3289f..5bc559b883b 100644--- a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb+++ b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb@@ -73,5 +73,41 @@ describe DiscourseAi::Translation::BaseTranslator do).to eq "hur dur hur dur!"endend++ it "sends the content to the model without HTML entity escaping" do+ html_text = "List<string> and ?a=1&b=2"+ post_translator =+ DiscourseAi::Translation::PostRawTranslator.new(text: html_text, target_locale:, post:)++ bot_context = instance_double(DiscourseAi::Agents::BotContext)+ allow(DiscourseAi::Agents::BotContext).to receive(:new).and_return(bot_context)+ mock_bot = instance_double(DiscourseAi::Agents::Bot)+ allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(mock_bot)+ allow(mock_bot).to receive(:reply).and_yield(llm_response, nil, nil)++ post_translator.translate++ expect(DiscourseAi::Agents::BotContext).to have_received(:new).with(+ user: an_instance_of(User),+ skip_show_thinking: true,+ feature_name: "translation",+ messages: [+ {+ type: :user,+ content: "{\"content\":\"List<string> and ?a=1&b=2\",\"target_locale\":\"de\"}",+ },+ ],+ topic: nil,+ post: post,+ )+ end++ it "strips control characters from the model response but keeps newlines" do+ DiscourseAi::Completions::Llm.with_prepared_responses(["hur\u001Cdur \u001Ehur\ndur!"]) do+ expect(+ DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:).translate,+ ).to eq "hurdur hur\ndur!"+ end+ endendend