Skip to content

A platform for community discussion. Free, open, simple.

UX: Continue after tool call approval in chat (#43661)

Penar Musaraj26a139a13f9a
diff --git a/plugins/discourse-ai/app/jobs/regular/resume_ai_tool_approval.rb b/plugins/discourse-ai/app/jobs/regular/resume_ai_tool_approval.rb
index 25f25c6996c..46ee10778db 100644
--- a/plugins/discourse-ai/app/jobs/regular/resume_ai_tool_approval.rb
+++ b/plugins/discourse-ai/app/jobs/regular/resume_ai_tool_approval.rb
@@ -13,15 +13,32 @@ module Jobs
continuation = reviewable.payload["continuation"]
return if continuation.blank?
- post = Post.includes(:topic).find_by(id: continuation["post_id"])
- return if !post || post.topic_id != reviewable.topic_id
+ if continuation["chat_message_id"]
+ return if !defined?(::Chat::Message) || !SiteSetting.chat_enabled
- authorization_user_id =
- post.custom_fields[
- DiscourseAi::AiBot::POST_AI_AGENT_AUTHORIZATION_USER_ID_FIELD
- ].presence || reviewable.target_post&.user_id
- user = User.find_by(id: authorization_user_id)
- return if !user || user.bot? || !user.guardian.can_see?(post)
+ approval_message = ::Chat::Message.find_by(id: continuation["chat_message_id"])
+ message = ::Chat::Message.find_by(id: reviewable.payload["chat_message_id"])
+ return if !approval_message || !message
+
+ channel = message.chat_channel
+ return if !channel.direct_message_channel? || approval_message.chat_channel_id != channel.id
+ return if approval_message.thread_id != message.thread_id
+
+ user = message.user
+ return if !user || user.bot? || !user.guardian.can_send_direct_message?(channel)
+ return if !user.guardian.can_create_chat_message?
+ return if !user.guardian.can_create_channel_message?(channel)
+ else
+ post = Post.includes(:topic).find_by(id: continuation["post_id"])
+ return if !post || post.topic_id != reviewable.topic_id
+
+ authorization_user_id =
+ post.custom_fields[
+ DiscourseAi::AiBot::POST_AI_AGENT_AUTHORIZATION_USER_ID_FIELD
+ ].presence || reviewable.target_post&.user_id
+ user = User.find_by(id: authorization_user_id)
+ return if !user || user.bot? || !user.guardian.can_see?(post)
+ end
tool_action = reviewable.target
return if !tool_action
@@ -30,6 +47,11 @@ module Jobs
bot_user = User.find_by(id: tool_action.bot_user_id)
return if !agent || !bot_user
+ if message
+ return if !agent.allow_chat_direct_messages
+ return if !channel.allowed_user_ids.include?(agent.user_id)
+ end
+
model = LlmModel.find_by(id: reviewable.payload["llm_model_id"])
bot = DiscourseAi::Agents::Bot.as(bot_user, agent: agent.new, model: model)
@@ -46,13 +68,7 @@ module Jobs
result: continuation["tool_result"],
}
- DiscourseAi::AiBot::Playground.new(bot).reply_to(
- post,
- attributed_user: user,
- authorization_user_id: user.id,
- feature_name: "bot",
- auto_set_title: false,
- custom_instructions: <<~TEXT.strip,
+ custom_instructions = <<~TEXT.strip
You are continuing after a human resolved a tool approval request.
The final message contains the decision and the actual tool result as JSON data.
If approved, the tool has already executed: use its result and do not execute it again.
@@ -61,8 +77,28 @@ module Jobs
or ask the user how to proceed if the rejected action was necessary.
Subsequent actions still require their normal approvals.
TEXT
- additional_messages: [{ type: :user, content: decision.to_json }],
- )
+ additional_messages = [{ type: :user, content: decision.to_json }]
+ playground = DiscourseAi::AiBot::Playground.new(bot)
+
+ if message
+ playground.reply_to_chat_message(
+ message,
+ channel,
+ reviewable.payload["context_post_ids"],
+ custom_instructions: custom_instructions,
+ additional_messages: additional_messages,
+ )
+ else
+ playground.reply_to(
+ post,
+ attributed_user: user,
+ authorization_user_id: user.id,
+ feature_name: "bot",
+ auto_set_title: false,
+ custom_instructions: custom_instructions,
+ additional_messages: additional_messages,
+ )
+ end
end
end
end
diff --git a/plugins/discourse-ai/app/models/reviewable_ai_tool_action.rb b/plugins/discourse-ai/app/models/reviewable_ai_tool_action.rb
index 35d5cfd1886..7ca09522333 100644
--- a/plugins/discourse-ai/app/models/reviewable_ai_tool_action.rb
+++ b/plugins/discourse-ai/app/models/reviewable_ai_tool_action.rb
@@ -42,6 +42,7 @@ class ReviewableAiToolAction < Reviewable
def perform_approve(performed_by, args)
ensure_inline_post_contains_approval!(args)
+ ensure_chat_message_contains_approval!(args)
ensure_performed_by_is_a_real_person!(performed_by)
tool, tool_class, context = build_tool!
@@ -63,6 +64,7 @@ class ReviewableAiToolAction < Reviewable
def perform_reject(performed_by, args)
ensure_inline_post_contains_approval!(args)
+ ensure_chat_message_contains_approval!(args)
ensure_performed_by_is_a_real_person!(performed_by)
resolution_result(:rejected, args)
@@ -73,9 +75,14 @@ class ReviewableAiToolAction < Reviewable
def resolution_result(status, args, tool_result: nil)
result = create_result(:success, status)
inline_post_id = args[:post_id] || args["post_id"]
- return result if inline_post_id.blank?
+ chat_message_id = args[:chat_message_id] || args["chat_message_id"]
+ return result if inline_post_id.blank? && chat_message_id.blank?
- payload["continuation"] = { post_id: inline_post_id.to_i, tool_result: tool_result }
+ payload["continuation"] = if chat_message_id.present?
+ { chat_message_id: chat_message_id.to_i, tool_result: tool_result }
+ else
+ { post_id: inline_post_id.to_i, tool_result: tool_result }
+ end
result.after_commit = -> do
DB.after_commit { Jobs.enqueue(:resume_ai_tool_approval, reviewable_id: id) }
end
@@ -145,6 +152,32 @@ class ReviewableAiToolAction < Reviewable
end
end
+ def ensure_chat_message_contains_approval!(args)
+ message_id = args[:chat_message_id] || args["chat_message_id"]
+ return if message_id.blank?
+
+ message = ::Chat::Message.find_by(id: message_id)
+ source = ::Chat::Message.find_by(id: payload["chat_message_id"])
+ action_id = DiscourseAi::AiBot::ChatToolApproval.build_action_id("approve", id)
+ valid_block =
+ message&.blocks.to_a.any? do |block|
+ block["elements"].to_a.any? { |element| element["action_id"] == action_id }
+ end
+
+ if !message || !message.chat_channel.direct_message_channel? ||
+ ![target&.bot_user_id, target&.ai_agent&.user_id].compact.include?(message.user_id) ||
+ !valid_block ||
+ (
+ source &&
+ (
+ source.chat_channel_id != message.chat_channel_id ||
+ source.thread_id != message.thread_id
+ )
+ )
+ raise Discourse::InvalidAccess
+ end
+ end
+
def ensure_performed_by_is_a_real_person!(performed_by)
if performed_by.blank? || performed_by.bot?
raise Discourse::InvalidAccess.new(
diff --git a/plugins/discourse-ai/lib/agents/bot.rb b/plugins/discourse-ai/lib/agents/bot.rb
index a37634e0960..4c0c6d2a180 100644
--- a/plugins/discourse-ai/lib/agents/bot.rb
+++ b/plugins/discourse-ai/lib/agents/bot.rb
@@ -503,6 +503,8 @@ module DiscourseAi
agent_name: @agent.class.name,
reason: tool.parameters[:reason],
llm_model_id: @model&.id,
+ chat_message_id: context.message_id,
+ context_post_ids: context.context_post_ids,
},
)
diff --git a/plugins/discourse-ai/lib/ai_bot/chat_tool_approval.rb b/plugins/discourse-ai/lib/ai_bot/chat_tool_approval.rb
index f932404c315..d7d45d8d971 100644
--- a/plugins/discourse-ai/lib/ai_bot/chat_tool_approval.rb
+++ b/plugins/discourse-ai/lib/ai_bot/chat_tool_approval.rb
@@ -77,7 +77,7 @@ module DiscourseAi
message = interaction.message
begin
- reviewable.perform(user, parsed[:action].to_sym)
+ reviewable.perform(user, parsed[:action].to_sym, chat_message_id: message.id)
status_key = parsed[:action] == "approve" ? "approved" : "rejected"
resolve_message!(
message,
diff --git a/plugins/discourse-ai/lib/ai_bot/playground.rb b/plugins/discourse-ai/lib/ai_bot/playground.rb
index 88079d0465f..d1111f1ea1a 100644
--- a/plugins/discourse-ai/lib/ai_bot/playground.rb
+++ b/plugins/discourse-ai/lib/ai_bot/playground.rb
@@ -339,7 +339,13 @@ module DiscourseAi
Discourse.warn_exception(e, message: "Discourse AI: Unable to generate title")
end
- def reply_to_chat_message(message, channel, context_post_ids)
+ def reply_to_chat_message(
+ message,
+ channel,
+ context_post_ids,
+ custom_instructions: nil,
+ additional_messages: []
+ )
agent_user = User.find(bot.agent.class.user_id)
participants = channel.user_chat_channel_memberships.map { |m| m.user.username }
@@ -355,6 +361,7 @@ module DiscourseAi
context =
DiscourseAi::Agents::BotContext.new(
participants: participants,
+ custom_instructions: custom_instructions,
message_id: message.id,
channel_id: channel.id,
context_post_ids: context_post_ids,
@@ -377,8 +384,10 @@ module DiscourseAi
cancel_manager: DiscourseAi::Completions::CancelManager.new,
)
+ context.messages.concat(additional_messages)
+
reply = nil
- guardian = Guardian.new(agent_user)
+ guardian = agent_user.guardian
force_thread = message.thread_id.nil? && channel.direct_message_channel?
in_reply_to_id = channel.direct_message_channel? ? message.id : nil
diff --git a/plugins/discourse-ai/spec/lib/ai_bot/chat_tool_approval_spec.rb b/plugins/discourse-ai/spec/lib/ai_bot/chat_tool_approval_spec.rb
index 625e52e9ce9..0bca35bfc83 100644
--- a/plugins/discourse-ai/spec/lib/ai_bot/chat_tool_approval_spec.rb
+++ b/plugins/discourse-ai/spec/lib/ai_bot/chat_tool_approval_spec.rb
@@ -136,6 +136,18 @@ RSpec.describe DiscourseAi::AiBot::ChatToolApproval do
expect(message.reload.blocks).to be_present
end
+ it "rejects an approval attached to a human message" do
+ reviewable = create_reviewable
+ message = message_for(reviewable)
+ message.update!(user: admin)
+
+ described_class.handle_interaction(interaction_for(reviewable, user: admin, message: message))
+
+ expect(reviewable.reload).to be_pending
+ expect(target_user.reload.suspended?).to eq(false)
+ expect(reviewable.payload["continuation"]).to be_nil
+ end
+
it "ignores foreign action ids" do
reviewable = create_reviewable
message = message_for(reviewable)
@@ -202,6 +214,10 @@ RSpec.describe DiscourseAi::AiBot::ChatToolApproval do
expect(target_user.reload.suspended?).to eq(true)
expect(reviewable.reload).not_to be_pending
expect(message.reload.blocks).to be_blank
+ expect(reviewable.payload["continuation"]).to include("chat_message_id" => message.id)
+ expect(Jobs::ResumeAiToolApproval.jobs.last["args"].first).to include(
+ "reviewable_id" => reviewable.id,
+ )
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/jobs/regular/resume_ai_tool_approval_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/jobs/regular/resume_ai_tool_approval_spec.rb
index 4c61429b081..731dce58af6 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/jobs/regular/resume_ai_tool_approval_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/jobs/regular/resume_ai_tool_approval_spec.rb
@@ -173,4 +173,170 @@ RSpec.describe Jobs::ResumeAiToolApproval do
expect(reviewable.reload.payload["continuation_started_at"]).to be_nil
end
end
+
+ describe "#execute in chat" do
+ before do
+ SiteSetting.chat_enabled = true
+ ai_agent.update!(allow_chat_direct_messages: true)
+ end
+
+ let(:channel) do
+ Fabricate(
+ :direct_message_channel,
+ users: [requester, admin, ai_agent.user],
+ threading_enabled: true,
+ )
+ end
+ let(:message) { Fabricate(:chat_message, chat_channel: channel, user: requester) }
+ let(:thread) { Fabricate(:chat_thread, channel: channel, original_message: message) }
+ let(:approval_message) do
+ message.update!(thread: thread)
+ tool_action.update!(post_id: nil)
+ reviewable.update!(payload: reviewable.payload.merge("chat_message_id" => message.id))
+ Fabricate(
+ :chat_message,
+ chat_channel: channel,
+ thread: thread,
+ user: ai_agent.user,
+ blocks: DiscourseAi::AiBot::ChatToolApproval.pending_blocks(reviewable.id),
+ )
+ end
+
+ it "resumes once in the same thread with the executed tool result" do
+ reviewable.perform(admin, :approve, chat_message_id: approval_message.id)
+ category = Category.find_by!(name: "Bug reports")
+ prompts = nil
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ ["The category is ready."],
+ ) do |_, _, captured_prompts|
+ job.execute(reviewable_id: reviewable.id)
+ job.execute(reviewable_id: reviewable.id)
+ prompts = captured_prompts
+ end
+
+ expect(prompts.size).to eq(1)
+ expect(JSON.parse(prompts.first.messages.last[:content])).to include(
+ "decision" => "approved",
+ "result" => include("category_id" => category.id),
+ )
+ reply = channel.chat_messages.order(:id).last
+ expect(reply.message).to eq("The category is ready.")
+ expect(reply.thread_id).to eq(thread.id)
+ expect(reply.user_id).to eq(ai_agent.user_id)
+ expect(Category.where(name: "Bug reports").count).to eq(1)
+ end
+
+ it "continues after rejection without executing the tool" do
+ reviewable.perform(admin, :reject, chat_message_id: approval_message.id)
+ prompts = nil
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ ["I will leave the categories unchanged."],
+ ) do |_, _, captured_prompts|
+ job.execute(reviewable_id: reviewable.id)
+ prompts = captured_prompts
+ end
+
+ expect(JSON.parse(prompts.first.messages.last[:content])).to include(
+ "decision" => "rejected",
+ "result" => nil,
+ )
+ expect(prompts.first.messages.first[:content]).to include("do not retry it")
+ expect(channel.chat_messages.order(:id).last.message).to eq(
+ "I will leave the categories unchanged.",
+ )
+ expect(Category.exists?(name: "Bug reports")).to eq(false)
+ end
+
+ it "queues chat approvals with the original context and resumes subsequent approvals" do
+ requester.update!(admin: true)
+ ai_agent.update!(tools: ["CreateCategory"])
+ bot =
+ DiscourseAi::Agents::Bot.as(
+ ai_agent.user,
+ agent: ai_agent.class_instance.new,
+ model: llm_model,
+ )
+ tool_calls =
+ ["First category", "Second category"].map do |name|
+ DiscourseAi::Completions::ToolCall.new(
+ name: "create_category",
+ id: name.parameterize,
+ parameters: {
+ name: name,
+ reason: "Organize discussions",
+ },
+ )
+ end
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [tool_calls.first, "Awaiting approval."],
+ ) do
+ DiscourseAi::AiBot::Playground.new(bot).reply_to_chat_message(
+ message,
+ channel,
+ [source_post.id],
+ )
+ end
+
+ first_reviewable = ReviewableAiToolAction.order(:id).last
+ expect(first_reviewable.payload).to include(
+ "chat_message_id" => message.id,
+ "context_post_ids" => [source_post.id],
+ )
+ first_reviewable.perform(
+ admin,
+ :approve,
+ chat_message_id: channel.chat_messages.order(:id).last.id,
+ )
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [tool_calls.last, "Awaiting another approval."],
+ ) { job.execute(reviewable_id: first_reviewable.id) }
+
+ next_reviewable = ReviewableAiToolAction.order(:id).last
+ expect(next_reviewable).to be_pending
+ expect(next_reviewable.payload["chat_message_id"]).to eq(message.id)
+ expect(Category.exists?(name: "Second category")).to eq(false)
+ next_reviewable.perform(
+ admin,
+ :approve,
+ chat_message_id: channel.chat_messages.order(:id).last.id,
+ )
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(["Both categories are ready."]) do
+ job.execute(reviewable_id: next_reviewable.id)
+ end
+
+ reply = channel.chat_messages.order(:id).last
+ expect(reply.message).to eq("Both categories are ready.")
+ expect(reply.thread_id).to eq(message.reload.thread_id)
+ expect(Category.where(name: ["First category", "Second category"]).count).to eq(2)
+ end
+
+ it "skips continuation when the original requester loses access" do
+ reviewable.perform(admin, :reject, chat_message_id: approval_message.id)
+ channel.chatable.direct_message_users.where(user_id: requester.id).delete_all
+
+ expect { job.execute(reviewable_id: reviewable.id) }.not_to change { Chat::Message.count }
+ expect(reviewable.reload.payload["continuation_started_at"]).to be_nil
+ end
+
+ it "skips continuation when the agent is no longer available to the requester" do
+ reviewable.perform(admin, :reject, chat_message_id: approval_message.id)
+ ai_agent.update!(allowed_group_ids: [Group::AUTO_GROUPS[:admins]])
+
+ expect { job.execute(reviewable_id: reviewable.id) }.not_to change { Chat::Message.count }
+ expect(reviewable.reload.payload["continuation_started_at"]).to be_nil
+ end
+
+ it "skips continuation when the source message is deleted" do
+ reviewable.perform(admin, :reject, chat_message_id: approval_message.id)
+ message.trash!
+
+ expect { job.execute(reviewable_id: reviewable.id) }.not_to change { Chat::Message.count }
+ expect(reviewable.reload.payload["continuation_started_at"]).to be_nil
+ end
+ end
end