A platform for community discussion. Free, open, simple.
SECURITY: escape LLM triage output in review flag reasons
Roman Rizzi8ae7cb2414d6
diff --git a/plugins/discourse-ai/lib/agents/tools/flag_post.rb b/plugins/discourse-ai/lib/agents/tools/flag_post.rbindex 0c9a4bbb3f6..d7de03086a1 100644--- a/plugins/discourse-ai/lib/agents/tools/flag_post.rb+++ b/plugins/discourse-ai/lib/agents/tools/flag_post.rb@@ -157,9 +157,10 @@ module DiscourseAiI18n.t("discourse_automation.scriptables.llm_triage.flagged_post",base_path: feature_context[:base_path] || Discourse.base_path,- llm_response: feature_context[:llm_response].presence || reason,+ llm_response:+ ERB::Util.html_escape(feature_context[:llm_response].presence || reason),automation_id: feature_context[:automation_id].to_s,- automation_name: feature_context[:automation_name].to_s,+ automation_name: ERB::Util.html_escape(feature_context[:automation_name].to_s),)elseI18n.t("discourse_ai.ai_bot.flag_post.reason", reason: reason)diff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rbindex 430d175460b..7568f5e131f 100644--- a/plugins/discourse-ai/lib/automation/llm_triage.rb+++ b/plugins/discourse-ai/lib/automation/llm_triage.rb@@ -191,9 +191,9 @@ module DiscourseAiI18n.t("discourse_automation.scriptables.llm_triage.flagged_post",base_path: Discourse.base_path,- llm_response: result,+ llm_response: ERB::Util.html_escape(result),automation_id: automation&.id.to_s,- automation_name: automation&.name.to_s,+ automation_name: ERB::Util.html_escape(automation&.name.to_s),)if !flagged_by_tooldiff --git a/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rbindex d8532477885..b9622e258f4 100644--- a/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rb+++ b/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rb@@ -88,4 +88,35 @@ RSpec.describe DiscourseAi::Agents::Tools::FlagPost doexpect(result[:status]).to eq("flagged")expect(post.reload).to be_hiddenend++ it "escapes llm response and automation name in automation flag reasons" do+ context =+ DiscourseAi::Agents::BotContext.new(+ post: post,+ feature_context: {+ automation_id: "123",+ automation_name: %(rule"><img src=x onerror=1>),+ llm_response: "<img src=x onerror=alert(1)>",+ base_path: Discourse.base_path,+ },+ )++ result =+ described_class.new(+ { flag_post: true, reason: "fallback reason" },+ bot_user: bot_user,+ llm: llm,+ context: context,+ agent_options: {+ },+ ).invoke++ expect(result[:status]).to eq("flagged")++ score_reason = ReviewablePost.last.reviewable_scores.first.reason+ expect(score_reason).to include("<img src=x onerror=alert(1)>")+ expect(score_reason).to include("rule"><img src=x onerror=1>")+ expect(score_reason).not_to include("<img src=x onerror=alert(1)>")+ expect(score_reason).not_to include(%(rule"><img src=x onerror=1>))+ endenddiff --git a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rbindex 457f67b5248..f8c4e5ff701 100644--- a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb@@ -418,6 +418,27 @@ describe DiscourseAi::Automation::LlmTriage do)end+ it "escapes llm response and automation name in the flagged post message" do+ automation = Fabricate(:automation, script: "llm_triage", name: %(rule"><img src=x onerror=1>))++ DiscourseAi::Completions::Llm.with_prepared_responses(["<img src=x onerror=alert(1)>"]) do+ triage(+ post: post,+ triage_agent_id: ai_agent.id,+ search_for_text: "img",+ flag_post: true,+ automation: automation,+ )+ end++ score_reason = ReviewablePost.last.reviewable_scores.first.reason++ expect(score_reason).to include("<img src=x onerror=alert(1)>")+ expect(score_reason).to include("rule"><img src=x onerror=1>")+ expect(score_reason).not_to include("<img src=x onerror=alert(1)>")+ expect(score_reason).not_to include(%(rule"><img src=x onerror=1>))+ end+it "only sends one PM when multiple rules flag the same post" do# First rule flags the post and sends PMDiscourseAi::Completions::Llm.with_prepared_responses(%w[bad bad]) do