Skip to content

A platform for community discussion. Free, open, simple.

SECURITY: escape LLM triage output in review flag reasons

Roman Rizzi8ae7cb2414d6
diff --git a/plugins/discourse-ai/lib/agents/tools/flag_post.rb b/plugins/discourse-ai/lib/agents/tools/flag_post.rb
index 0c9a4bbb3f6..d7de03086a1 100644
--- a/plugins/discourse-ai/lib/agents/tools/flag_post.rb
+++ b/plugins/discourse-ai/lib/agents/tools/flag_post.rb
@@ -157,9 +157,10 @@ module DiscourseAi
I18n.t(
"discourse_automation.scriptables.llm_triage.flagged_post",
base_path: feature_context[:base_path] || Discourse.base_path,
- llm_response: feature_context[:llm_response].presence || reason,
+ llm_response:
+ ERB::Util.html_escape(feature_context[:llm_response].presence || reason),
automation_id: feature_context[:automation_id].to_s,
- automation_name: feature_context[:automation_name].to_s,
+ automation_name: ERB::Util.html_escape(feature_context[:automation_name].to_s),
)
else
I18n.t("discourse_ai.ai_bot.flag_post.reason", reason: reason)
diff --git a/plugins/discourse-ai/lib/automation/llm_triage.rb b/plugins/discourse-ai/lib/automation/llm_triage.rb
index 430d175460b..7568f5e131f 100644
--- a/plugins/discourse-ai/lib/automation/llm_triage.rb
+++ b/plugins/discourse-ai/lib/automation/llm_triage.rb
@@ -191,9 +191,9 @@ module DiscourseAi
I18n.t(
"discourse_automation.scriptables.llm_triage.flagged_post",
base_path: Discourse.base_path,
- llm_response: result,
+ llm_response: ERB::Util.html_escape(result),
automation_id: automation&.id.to_s,
- automation_name: automation&.name.to_s,
+ automation_name: ERB::Util.html_escape(automation&.name.to_s),
)
if !flagged_by_tool
diff --git a/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rb b/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rb
index d8532477885..b9622e258f4 100644
--- a/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rb
+++ b/plugins/discourse-ai/spec/lib/agents/tools/flag_post_spec.rb
@@ -88,4 +88,35 @@ RSpec.describe DiscourseAi::Agents::Tools::FlagPost do
expect(result[:status]).to eq("flagged")
expect(post.reload).to be_hidden
end
+
+ it "escapes llm response and automation name in automation flag reasons" do
+ context =
+ DiscourseAi::Agents::BotContext.new(
+ post: post,
+ feature_context: {
+ automation_id: "123",
+ automation_name: %(rule"><img src=x onerror=1>),
+ llm_response: "<img src=x onerror=alert(1)>",
+ base_path: Discourse.base_path,
+ },
+ )
+
+ result =
+ described_class.new(
+ { flag_post: true, reason: "fallback reason" },
+ bot_user: bot_user,
+ llm: llm,
+ context: context,
+ agent_options: {
+ },
+ ).invoke
+
+ expect(result[:status]).to eq("flagged")
+
+ score_reason = ReviewablePost.last.reviewable_scores.first.reason
+ expect(score_reason).to include("&lt;img src=x onerror=alert(1)&gt;")
+ expect(score_reason).to include("rule&quot;&gt;&lt;img src=x onerror=1&gt;")
+ expect(score_reason).not_to include("<img src=x onerror=alert(1)>")
+ expect(score_reason).not_to include(%(rule"><img src=x onerror=1>))
+ end
end
diff --git a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
index 457f67b5248..f8c4e5ff701 100644
--- a/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/automation/llm_triage_spec.rb
@@ -418,6 +418,27 @@ describe DiscourseAi::Automation::LlmTriage do
)
end
+ it "escapes llm response and automation name in the flagged post message" do
+ automation = Fabricate(:automation, script: "llm_triage", name: %(rule"><img src=x onerror=1>))
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["<img src=x onerror=alert(1)>"]) do
+ triage(
+ post: post,
+ triage_agent_id: ai_agent.id,
+ search_for_text: "img",
+ flag_post: true,
+ automation: automation,
+ )
+ end
+
+ score_reason = ReviewablePost.last.reviewable_scores.first.reason
+
+ expect(score_reason).to include("&lt;img src=x onerror=alert(1)&gt;")
+ expect(score_reason).to include("rule&quot;&gt;&lt;img src=x onerror=1&gt;")
+ expect(score_reason).not_to include("<img src=x onerror=alert(1)>")
+ expect(score_reason).not_to include(%(rule"><img src=x onerror=1>))
+ end
+
it "only sends one PM when multiple rules flag the same post" do
# First rule flags the post and sends PM
DiscourseAi::Completions::Llm.with_prepared_responses(%w[bad bad]) do