Skip to content

A platform for community discussion. Free, open, simple.

FIX: Control characters injected into AI translations (#41743)

Rafael dos Santos Silvac0334fa7a32f
diff --git a/plugins/discourse-ai/lib/translation/base_translator.rb b/plugins/discourse-ai/lib/translation/base_translator.rb
index 901c929a7a4..43ba682bc8d 100644
--- a/plugins/discourse-ai/lib/translation/base_translator.rb
+++ b/plugins/discourse-ai/lib/translation/base_translator.rb
@@ -26,16 +26,28 @@ module DiscourseAi
bot = DiscourseAi::Agents::Bot.as(translation_user, agent:, model:)
- ContentSplitter
- .split(content: @text, chunk_size: model.max_output_tokens)
- .map { |text| get_translation(text:, bot:, translation_user:, model:) }
- .join("")
+ translated =
+ ContentSplitter
+ .split(content: @text, chunk_size: model.max_output_tokens)
+ .map { |text| get_translation(text:, bot:, translation_user:, model:) }
+ .join("")
+
+ strip_control_characters(translated)
end
private
def formatted_content(content)
- { content:, target_locale: @target_locale }.to_json
+ # JSON.generate over to_json: ActiveSupport HTML-escapes <, >, and & into
+ # \uXXXX sequences, which models can mis-copy into control characters
+ JSON.generate({ content:, target_locale: @target_locale })
+ end
+
+ # control characters are never valid in a translation, but models
+ # occasionally emit them by mangling unicode escapes (e.g. \u003c
+ # coming back as \u001c)
+ def strip_control_characters(text)
+ text.gsub(/[\u0000-\u0008\u000B-\u001F\u007F\u0080-\u009F]/, "")
end
def get_translation(text:, bot:, translation_user:, model:)
diff --git a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
index 21ce0e3289f..5bc559b883b 100644
--- a/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
+++ b/plugins/discourse-ai/spec/lib/translation/base_translator_spec.rb
@@ -73,5 +73,41 @@ describe DiscourseAi::Translation::BaseTranslator do
).to eq "hur dur hur dur!"
end
end
+
+ it "sends the content to the model without HTML entity escaping" do
+ html_text = "List<string> and ?a=1&b=2"
+ post_translator =
+ DiscourseAi::Translation::PostRawTranslator.new(text: html_text, target_locale:, post:)
+
+ bot_context = instance_double(DiscourseAi::Agents::BotContext)
+ allow(DiscourseAi::Agents::BotContext).to receive(:new).and_return(bot_context)
+ mock_bot = instance_double(DiscourseAi::Agents::Bot)
+ allow(DiscourseAi::Agents::Bot).to receive(:as).and_return(mock_bot)
+ allow(mock_bot).to receive(:reply).and_yield(llm_response, nil, nil)
+
+ post_translator.translate
+
+ expect(DiscourseAi::Agents::BotContext).to have_received(:new).with(
+ user: an_instance_of(User),
+ skip_show_thinking: true,
+ feature_name: "translation",
+ messages: [
+ {
+ type: :user,
+ content: "{\"content\":\"List<string> and ?a=1&b=2\",\"target_locale\":\"de\"}",
+ },
+ ],
+ topic: nil,
+ post: post,
+ )
+ end
+
+ it "strips control characters from the model response but keeps newlines" do
+ DiscourseAi::Completions::Llm.with_prepared_responses(["hur\u001Cdur \u001Ehur\ndur!"]) do
+ expect(
+ DiscourseAi::Translation::PostRawTranslator.new(text:, target_locale:).translate,
+ ).to eq "hurdur hur\ndur!"
+ end
+ end
end
end