Skip to content

A platform for community discussion. Free, open, simple.

FIX: Embed user IDs in vLLM prompts (#42295)

Sam11d2abf27733
diff --git a/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb b/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb
index 171b6d67455..7b09df9af88 100644
--- a/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb
@@ -114,11 +114,11 @@ module DiscourseAi
user_message = { role: }
- if msg[:id]
+ if (user_id = user_id_for(msg))
if embed_user_ids?
- content_array << "#{msg[:id]}: "
+ content_array << "#{user_id}: "
else
- user_message[:name] = msg[:id]
+ user_message[:name] = user_id
end
end
diff --git a/plugins/discourse-ai/lib/completions/dialects/claude.rb b/plugins/discourse-ai/lib/completions/dialects/claude.rb
index bea92ebb05b..953f694915b 100644
--- a/plugins/discourse-ai/lib/completions/dialects/claude.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/claude.rb
@@ -169,7 +169,9 @@ module DiscourseAi
def user_msg(msg)
content_array = []
- content_array << "#{msg[:id]}: " if msg[:id]
+ if (prefix = user_id_prefix(msg))
+ content_array << prefix
+ end
content_array.concat([msg[:content]].flatten)
content_array =
diff --git a/plugins/discourse-ai/lib/completions/dialects/command.rb b/plugins/discourse-ai/lib/completions/dialects/command.rb
index 55b9ecb45ff..66e06da970a 100644
--- a/plugins/discourse-ai/lib/completions/dialects/command.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/command.rb
@@ -110,7 +110,9 @@ module DiscourseAi
def user_msg(msg)
content = DiscourseAi::Completions::Prompt.text_only(msg)
user_message = { role: "USER", message: content }
- user_message[:message] = "#{msg[:id]}: #{content}" if msg[:id]
+ if (user_id = user_id_for(msg))
+ user_message[:message] = "#{user_id}: #{content}"
+ end
user_message
end
end
diff --git a/plugins/discourse-ai/lib/completions/dialects/converse.rb b/plugins/discourse-ai/lib/completions/dialects/converse.rb
index 54575f790e1..1d27ec11a4c 100644
--- a/plugins/discourse-ai/lib/completions/dialects/converse.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/converse.rb
@@ -107,7 +107,7 @@ module DiscourseAi
end
def user_msg(msg)
- content_array = [msg[:content]].flatten
+ content_array = [prepend_user_id(msg[:content], msg)].flatten
content_array =
to_encoded_content_array(
diff --git a/plugins/discourse-ai/lib/completions/dialects/dialect.rb b/plugins/discourse-ai/lib/completions/dialects/dialect.rb
index 20460fd22e9..1f97b2f117a 100644
--- a/plugins/discourse-ai/lib/completions/dialects/dialect.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/dialect.rb
@@ -202,6 +202,30 @@ module DiscourseAi
attr_reader :opts, :llm_model
+ def user_id_for(msg)
+ msg[:id] if msg[:type].to_sym == :user && msg[:id].present?
+ end
+
+ def user_id_prefix(msg)
+ user_id = user_id_for(msg)
+ "#{user_id}: " if user_id
+ end
+
+ def prepend_user_id(content, msg)
+ prefix = user_id_prefix(msg)
+ return content unless prefix
+ return "#{prefix}#{content}" unless content.is_a?(Array)
+
+ content = content.dup
+ first_text_index = content.index { |item| item.is_a?(String) }
+ if first_text_index
+ content[first_text_index] = "#{prefix}#{content[first_text_index]}"
+ else
+ content.unshift(prefix)
+ end
+ content
+ end
+
def strip_upload_markers(markdown, upload_shas)
return markdown if markdown.blank? || upload_shas.blank?
base62_set = upload_shas.compact.map { |sha| Upload.base62_sha1(sha) }.to_set
diff --git a/plugins/discourse-ai/lib/completions/dialects/gemini.rb b/plugins/discourse-ai/lib/completions/dialects/gemini.rb
index 80967d8ed5d..a8f5f7837bc 100644
--- a/plugins/discourse-ai/lib/completions/dialects/gemini.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/gemini.rb
@@ -119,7 +119,9 @@ module DiscourseAi
def message_for_role(role, msg)
content_array = []
- content_array << "#{msg[:id]}: " if msg[:id]
+ if (prefix = user_id_prefix(msg))
+ content_array << prefix
+ end
content_array << msg[:content]
content_array.flatten!
diff --git a/plugins/discourse-ai/lib/completions/dialects/nova.rb b/plugins/discourse-ai/lib/completions/dialects/nova.rb
index 0db7f116f3c..28af248a51a 100644
--- a/plugins/discourse-ai/lib/completions/dialects/nova.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/nova.rb
@@ -140,6 +140,8 @@ module DiscourseAi
end
def user_msg(msg)
+ content = prepend_user_id(DiscourseAi::Completions::Prompt.text_only(msg), msg)
+
images = nil
if vision_support?
encoded_uploads = prompt.encoded_uploads(msg)
@@ -156,7 +158,7 @@ module DiscourseAi
end
end
- { role: "user", content: DiscourseAi::Completions::Prompt.text_only(msg), images: images }
+ { role: "user", content: content, images: images }
end
def model_msg(msg)
diff --git a/plugins/discourse-ai/lib/completions/dialects/ollama.rb b/plugins/discourse-ai/lib/completions/dialects/ollama.rb
index 60d58455c74..4102e0227ac 100644
--- a/plugins/discourse-ai/lib/completions/dialects/ollama.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/ollama.rb
@@ -69,7 +69,8 @@ module DiscourseAi
end
def user_msg(msg)
- user_message = { role: "user", content: DiscourseAi::Completions::Prompt.text_only(msg) }
+ content = prepend_user_id(DiscourseAi::Completions::Prompt.text_only(msg), msg)
+ user_message = { role: "user", content: content }
encoded_uploads = prompt.encoded_uploads(msg)
if encoded_uploads.present?
@@ -87,8 +88,6 @@ module DiscourseAi
user_message[:images] = images if images.present?
end
- # TODO: Add support for user messages with embedded user ids
-
user_message
end
end
diff --git a/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rb b/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rb
index d6519822c36..c64be19a0fb 100644
--- a/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rb
@@ -19,6 +19,10 @@ module DiscourseAi
@tools ||= tools_dialect.translated_tools
end
+ def embed_user_ids?
+ true
+ end
+
def max_prompt_tokens
return llm_model.max_prompt_tokens if llm_model&.max_prompt_tokens
diff --git a/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rb b/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rb
index dea4aed5795..e5eca6ac371 100644
--- a/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rb
@@ -97,7 +97,9 @@ module DiscourseAi
user_message = { role: }
- content_array << "#{msg[:id]}: " if msg[:id]
+ if (prefix = user_id_prefix(msg))
+ content_array << prefix
+ end
content_array << msg[:content]
diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb
index 2cf3c4545c0..42e7a733803 100644
--- a/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb
@@ -6,6 +6,42 @@ RSpec.describe DiscourseAi::Completions::Dialects::Converse do
before { enable_current_plugin }
describe "#translate" do
+ it "embeds participant names in user message content" do
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ nil,
+ messages: [{ type: :user, id: "admin", content: "Who am I?" }],
+ )
+
+ translated = described_class.new(prompt, model).translate
+
+ expect(translated.messages).to eq([{ role: "user", content: [{ text: "admin: Who am I?" }] }])
+ end
+
+ it "keeps participant names with text that follows an image" do
+ model.update!(vision_enabled: true)
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ nil,
+ messages: [
+ { type: :user, id: "admin", content: [{ upload_id: 123 }, "Describe this image"] },
+ ],
+ )
+
+ allow(DiscourseAi::Completions::UploadEncoder).to receive(:encode).and_return(
+ [{ kind: :image, mime_type: "image/png", base64: Base64.strict_encode64("image") }],
+ )
+
+ translated = described_class.new(prompt, model).translate
+
+ expect(translated.messages.first[:content]).to eq(
+ [
+ { image: { format: "png", source: { bytes: "image" } } },
+ { text: "admin: Describe this image" },
+ ],
+ )
+ end
+
it "renders converted document uploads as text content blocks" do
model.update!(allowed_attachment_types: ["docx"])
converted_text = "Uploaded document: sample.docx (13 Bytes)\n\nConverted text"
diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rb
index ad4d3f5aaee..ee7dc158843 100644
--- a/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rb
@@ -25,7 +25,7 @@ RSpec.describe DiscourseAi::Completions::Dialects::Nova do
expect(translated.system).to eq([{ text: "You are a helpful bot" }])
expect(translated.messages).to eq(
[
- { role: "user", content: [{ text: "Hello" }] },
+ { role: "user", content: [{ text: "user1: Hello" }] },
{ role: "assistant", content: [{ text: "Hi there!" }] },
],
)
@@ -55,7 +55,7 @@ RSpec.describe DiscourseAi::Completions::Dialects::Nova do
expect(translated.messages.first[:content]).to eq(
[
- { text: "What's in this image?" },
+ { text: "user1: What's in this image?" },
{ image: { format: "jpeg", source: { bytes: encoded } } },
],
)
diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rb
index 22db5028c9e..09ea057d839 100644
--- a/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rb
@@ -10,6 +10,30 @@ RSpec.describe DiscourseAi::Completions::Dialects::Ollama do
before { enable_current_plugin }
describe "#translate" do
+ it "embeds participant names in user message content" do
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ nil,
+ messages: [{ type: :user, id: "admin", content: "Who am I?" }],
+ )
+
+ expect(dialect_class.new(prompt, model).translate).to eq(
+ [{ role: "user", content: "admin: Who am I?" }],
+ )
+ end
+
+ it "does not embed blank participant names" do
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ nil,
+ messages: [{ type: :user, id: "", content: "Who am I?" }],
+ )
+
+ expect(dialect_class.new(prompt, model).translate).to eq(
+ [{ role: "user", content: "Who am I?" }],
+ )
+ end
+
context "when native tool support is enabled" do
it "translates a prompt written in our generic format to the Ollama format" do
ollama_version = [
@@ -92,19 +116,19 @@ RSpec.describe DiscourseAi::Completions::Dialects::Ollama do
model.update!(provider_params: { enable_native_tool: false })
tool = { name: "noop", description: "do nothing" }
+ tool_id = "tool-1"
messages = [
{ type: :user, content: "echo away" },
- { type: :tool_call, content: "{}", name: "noop" },
- { type: :tool, content: "{}", name: "noop" },
+ { type: :tool_call, id: tool_id, content: "{}", name: "noop" },
+ { type: :tool, id: tool_id, content: "{}", name: "noop" },
]
prompt = DiscourseAi::Completions::Prompt.new("a bot", tools: [tool], messages: messages)
dialect = dialect_class.new(prompt, model)
- expected = %w[system user assistant user]
- roles = dialect.translate.map { |x| x[:role] }
+ translated = dialect.translate
- # notice, no tool role
- expect(roles).to eq(expected)
+ expect(translated.map { |message| message[:role] }).to eq(%w[system user assistant user])
+ expect(translated.map { |message| message[:content] }.join).not_to include(tool_id)
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rb
index 1e66b6eb526..7eeb473e8cf 100644
--- a/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rb
@@ -74,6 +74,49 @@ RSpec.describe DiscourseAi::Completions::Dialects::OpenAiCompatible do
end
context "when system prompts are enabled" do
+ it "embeds participant names in user message content" do
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System instructions",
+ messages: [{ type: :user, id: "admin", content: "Who am I?" }],
+ )
+ model = Fabricate(:samba_nova_model, provider_params: { disable_system_prompt: false })
+
+ expect(described_class.new(prompt, model).translate).to eq(
+ [
+ { role: "system", content: "System instructions" },
+ { role: "user", content: "admin: Who am I?" },
+ ],
+ )
+ end
+
+ it "does not embed protocol IDs from XML tool messages" do
+ tool_id = "tool-1"
+ model =
+ Fabricate(
+ :samba_nova_model,
+ provider_params: {
+ disable_system_prompt: false,
+ disable_native_tools: true,
+ },
+ )
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System instructions",
+ tools: [{ name: "echo", description: "Echo the input" }],
+ messages: [
+ { type: :user, id: "admin", content: "Echo this" },
+ { type: :tool_call, id: tool_id, name: "echo", content: "{}" },
+ { type: :tool, id: tool_id, name: "echo", content: '"result"' },
+ ],
+ )
+
+ translated = described_class.new(prompt, model).translate
+
+ expect(translated.second[:content]).to start_with("admin: ")
+ expect(translated.map { |message| message[:content] }.join).not_to include(tool_id)
+ end
+
it "includes system and user messages separately" do
system_msg = "This is a system message"
user_msg = "user message"
diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb
index d6b22f648ac..677cb495be6 100644
--- a/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb
@@ -13,6 +13,27 @@ RSpec.describe DiscourseAi::Completions::Dialects::Vllm do
)
end
+ it "embeds participant names in user message content" do
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System instructions",
+ messages: [
+ { type: :user, id: "user1", content: "First question" },
+ { type: :model, content: "First answer" },
+ { type: :user, id: "admin", content: "Who am I?" },
+ ],
+ )
+
+ expect(described_class.new(prompt, model).translate).to eq(
+ [
+ { role: "system", content: "System instructions" },
+ { role: "user", content: "user1: First question" },
+ { role: "assistant", content: "First answer" },
+ { role: "user", content: "admin: Who am I?" },
+ ],
+ )
+ end
+
it "replays tool-call reasoning without adding reasoning to normal assistant messages" do
prompt =
DiscourseAi::Completions::Prompt.new(