A platform for community discussion. Free, open, simple.
FIX: Embed user IDs in vLLM prompts (#42295)
Sam11d2abf27733
diff --git a/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb b/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rbindex 171b6d67455..7b09df9af88 100644--- a/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb+++ b/plugins/discourse-ai/lib/completions/dialects/chat_gpt.rb@@ -114,11 +114,11 @@ module DiscourseAiuser_message = { role: }- if msg[:id]+ if (user_id = user_id_for(msg))if embed_user_ids?- content_array << "#{msg[:id]}: "+ content_array << "#{user_id}: "else- user_message[:name] = msg[:id]+ user_message[:name] = user_idendenddiff --git a/plugins/discourse-ai/lib/completions/dialects/claude.rb b/plugins/discourse-ai/lib/completions/dialects/claude.rbindex bea92ebb05b..953f694915b 100644--- a/plugins/discourse-ai/lib/completions/dialects/claude.rb+++ b/plugins/discourse-ai/lib/completions/dialects/claude.rb@@ -169,7 +169,9 @@ module DiscourseAidef user_msg(msg)content_array = []- content_array << "#{msg[:id]}: " if msg[:id]+ if (prefix = user_id_prefix(msg))+ content_array << prefix+ endcontent_array.concat([msg[:content]].flatten)content_array =diff --git a/plugins/discourse-ai/lib/completions/dialects/command.rb b/plugins/discourse-ai/lib/completions/dialects/command.rbindex 55b9ecb45ff..66e06da970a 100644--- a/plugins/discourse-ai/lib/completions/dialects/command.rb+++ b/plugins/discourse-ai/lib/completions/dialects/command.rb@@ -110,7 +110,9 @@ module DiscourseAidef user_msg(msg)content = DiscourseAi::Completions::Prompt.text_only(msg)user_message = { role: "USER", message: content }- user_message[:message] = "#{msg[:id]}: #{content}" if msg[:id]+ if (user_id = user_id_for(msg))+ user_message[:message] = "#{user_id}: #{content}"+ enduser_messageendenddiff --git a/plugins/discourse-ai/lib/completions/dialects/converse.rb b/plugins/discourse-ai/lib/completions/dialects/converse.rbindex 54575f790e1..1d27ec11a4c 100644--- a/plugins/discourse-ai/lib/completions/dialects/converse.rb+++ b/plugins/discourse-ai/lib/completions/dialects/converse.rb@@ -107,7 +107,7 @@ module DiscourseAienddef user_msg(msg)- content_array = [msg[:content]].flatten+ content_array = [prepend_user_id(msg[:content], msg)].flattencontent_array =to_encoded_content_array(diff --git a/plugins/discourse-ai/lib/completions/dialects/dialect.rb b/plugins/discourse-ai/lib/completions/dialects/dialect.rbindex 20460fd22e9..1f97b2f117a 100644--- a/plugins/discourse-ai/lib/completions/dialects/dialect.rb+++ b/plugins/discourse-ai/lib/completions/dialects/dialect.rb@@ -202,6 +202,30 @@ module DiscourseAiattr_reader :opts, :llm_model+ def user_id_for(msg)+ msg[:id] if msg[:type].to_sym == :user && msg[:id].present?+ end++ def user_id_prefix(msg)+ user_id = user_id_for(msg)+ "#{user_id}: " if user_id+ end++ def prepend_user_id(content, msg)+ prefix = user_id_prefix(msg)+ return content unless prefix+ return "#{prefix}#{content}" unless content.is_a?(Array)++ content = content.dup+ first_text_index = content.index { |item| item.is_a?(String) }+ if first_text_index+ content[first_text_index] = "#{prefix}#{content[first_text_index]}"+ else+ content.unshift(prefix)+ end+ content+ end+def strip_upload_markers(markdown, upload_shas)return markdown if markdown.blank? || upload_shas.blank?base62_set = upload_shas.compact.map { |sha| Upload.base62_sha1(sha) }.to_setdiff --git a/plugins/discourse-ai/lib/completions/dialects/gemini.rb b/plugins/discourse-ai/lib/completions/dialects/gemini.rbindex 80967d8ed5d..a8f5f7837bc 100644--- a/plugins/discourse-ai/lib/completions/dialects/gemini.rb+++ b/plugins/discourse-ai/lib/completions/dialects/gemini.rb@@ -119,7 +119,9 @@ module DiscourseAidef message_for_role(role, msg)content_array = []- content_array << "#{msg[:id]}: " if msg[:id]+ if (prefix = user_id_prefix(msg))+ content_array << prefix+ endcontent_array << msg[:content]content_array.flatten!diff --git a/plugins/discourse-ai/lib/completions/dialects/nova.rb b/plugins/discourse-ai/lib/completions/dialects/nova.rbindex 0db7f116f3c..28af248a51a 100644--- a/plugins/discourse-ai/lib/completions/dialects/nova.rb+++ b/plugins/discourse-ai/lib/completions/dialects/nova.rb@@ -140,6 +140,8 @@ module DiscourseAienddef user_msg(msg)+ content = prepend_user_id(DiscourseAi::Completions::Prompt.text_only(msg), msg)+images = nilif vision_support?encoded_uploads = prompt.encoded_uploads(msg)@@ -156,7 +158,7 @@ module DiscourseAiendend- { role: "user", content: DiscourseAi::Completions::Prompt.text_only(msg), images: images }+ { role: "user", content: content, images: images }enddef model_msg(msg)diff --git a/plugins/discourse-ai/lib/completions/dialects/ollama.rb b/plugins/discourse-ai/lib/completions/dialects/ollama.rbindex 60d58455c74..4102e0227ac 100644--- a/plugins/discourse-ai/lib/completions/dialects/ollama.rb+++ b/plugins/discourse-ai/lib/completions/dialects/ollama.rb@@ -69,7 +69,8 @@ module DiscourseAienddef user_msg(msg)- user_message = { role: "user", content: DiscourseAi::Completions::Prompt.text_only(msg) }+ content = prepend_user_id(DiscourseAi::Completions::Prompt.text_only(msg), msg)+ user_message = { role: "user", content: content }encoded_uploads = prompt.encoded_uploads(msg)if encoded_uploads.present?@@ -87,8 +88,6 @@ module DiscourseAiuser_message[:images] = images if images.present?end- # TODO: Add support for user messages with embedded user ids-user_messageendenddiff --git a/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rb b/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rbindex d6519822c36..c64be19a0fb 100644--- a/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rb+++ b/plugins/discourse-ai/lib/completions/dialects/open_ai_compatible.rb@@ -19,6 +19,10 @@ module DiscourseAi@tools ||= tools_dialect.translated_toolsend+ def embed_user_ids?+ true+ end+def max_prompt_tokensreturn llm_model.max_prompt_tokens if llm_model&.max_prompt_tokensdiff --git a/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rb b/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rbindex dea4aed5795..e5eca6ac371 100644--- a/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rb+++ b/plugins/discourse-ai/lib/completions/dialects/open_ai_responses.rb@@ -97,7 +97,9 @@ module DiscourseAiuser_message = { role: }- content_array << "#{msg[:id]}: " if msg[:id]+ if (prefix = user_id_prefix(msg))+ content_array << prefix+ endcontent_array << msg[:content]diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rbindex 2cf3c4545c0..42e7a733803 100644--- a/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/dialects/converse_spec.rb@@ -6,6 +6,42 @@ RSpec.describe DiscourseAi::Completions::Dialects::Converse dobefore { enable_current_plugin }describe "#translate" do+ it "embeds participant names in user message content" do+ prompt =+ DiscourseAi::Completions::Prompt.new(+ nil,+ messages: [{ type: :user, id: "admin", content: "Who am I?" }],+ )++ translated = described_class.new(prompt, model).translate++ expect(translated.messages).to eq([{ role: "user", content: [{ text: "admin: Who am I?" }] }])+ end++ it "keeps participant names with text that follows an image" do+ model.update!(vision_enabled: true)+ prompt =+ DiscourseAi::Completions::Prompt.new(+ nil,+ messages: [+ { type: :user, id: "admin", content: [{ upload_id: 123 }, "Describe this image"] },+ ],+ )++ allow(DiscourseAi::Completions::UploadEncoder).to receive(:encode).and_return(+ [{ kind: :image, mime_type: "image/png", base64: Base64.strict_encode64("image") }],+ )++ translated = described_class.new(prompt, model).translate++ expect(translated.messages.first[:content]).to eq(+ [+ { image: { format: "png", source: { bytes: "image" } } },+ { text: "admin: Describe this image" },+ ],+ )+ end+it "renders converted document uploads as text content blocks" domodel.update!(allowed_attachment_types: ["docx"])converted_text = "Uploaded document: sample.docx (13 Bytes)\n\nConverted text"diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rbindex ad4d3f5aaee..ee7dc158843 100644--- a/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/dialects/nova_spec.rb@@ -25,7 +25,7 @@ RSpec.describe DiscourseAi::Completions::Dialects::Nova doexpect(translated.system).to eq([{ text: "You are a helpful bot" }])expect(translated.messages).to eq([- { role: "user", content: [{ text: "Hello" }] },+ { role: "user", content: [{ text: "user1: Hello" }] },{ role: "assistant", content: [{ text: "Hi there!" }] },],)@@ -55,7 +55,7 @@ RSpec.describe DiscourseAi::Completions::Dialects::Nova doexpect(translated.messages.first[:content]).to eq([- { text: "What's in this image?" },+ { text: "user1: What's in this image?" },{ image: { format: "jpeg", source: { bytes: encoded } } },],)diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rbindex 22db5028c9e..09ea057d839 100644--- a/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/dialects/ollama_spec.rb@@ -10,6 +10,30 @@ RSpec.describe DiscourseAi::Completions::Dialects::Ollama dobefore { enable_current_plugin }describe "#translate" do+ it "embeds participant names in user message content" do+ prompt =+ DiscourseAi::Completions::Prompt.new(+ nil,+ messages: [{ type: :user, id: "admin", content: "Who am I?" }],+ )++ expect(dialect_class.new(prompt, model).translate).to eq(+ [{ role: "user", content: "admin: Who am I?" }],+ )+ end++ it "does not embed blank participant names" do+ prompt =+ DiscourseAi::Completions::Prompt.new(+ nil,+ messages: [{ type: :user, id: "", content: "Who am I?" }],+ )++ expect(dialect_class.new(prompt, model).translate).to eq(+ [{ role: "user", content: "Who am I?" }],+ )+ end+context "when native tool support is enabled" doit "translates a prompt written in our generic format to the Ollama format" doollama_version = [@@ -92,19 +116,19 @@ RSpec.describe DiscourseAi::Completions::Dialects::Ollama domodel.update!(provider_params: { enable_native_tool: false })tool = { name: "noop", description: "do nothing" }+ tool_id = "tool-1"messages = [{ type: :user, content: "echo away" },- { type: :tool_call, content: "{}", name: "noop" },- { type: :tool, content: "{}", name: "noop" },+ { type: :tool_call, id: tool_id, content: "{}", name: "noop" },+ { type: :tool, id: tool_id, content: "{}", name: "noop" },]prompt = DiscourseAi::Completions::Prompt.new("a bot", tools: [tool], messages: messages)dialect = dialect_class.new(prompt, model)- expected = %w[system user assistant user]- roles = dialect.translate.map { |x| x[:role] }+ translated = dialect.translate- # notice, no tool role- expect(roles).to eq(expected)+ expect(translated.map { |message| message[:role] }).to eq(%w[system user assistant user])+ expect(translated.map { |message| message[:content] }.join).not_to include(tool_id)endendenddiff --git a/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rbindex 1e66b6eb526..7eeb473e8cf 100644--- a/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/dialects/open_ai_compatible_spec.rb@@ -74,6 +74,49 @@ RSpec.describe DiscourseAi::Completions::Dialects::OpenAiCompatible doendcontext "when system prompts are enabled" do+ it "embeds participant names in user message content" do+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "System instructions",+ messages: [{ type: :user, id: "admin", content: "Who am I?" }],+ )+ model = Fabricate(:samba_nova_model, provider_params: { disable_system_prompt: false })++ expect(described_class.new(prompt, model).translate).to eq(+ [+ { role: "system", content: "System instructions" },+ { role: "user", content: "admin: Who am I?" },+ ],+ )+ end++ it "does not embed protocol IDs from XML tool messages" do+ tool_id = "tool-1"+ model =+ Fabricate(+ :samba_nova_model,+ provider_params: {+ disable_system_prompt: false,+ disable_native_tools: true,+ },+ )+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "System instructions",+ tools: [{ name: "echo", description: "Echo the input" }],+ messages: [+ { type: :user, id: "admin", content: "Echo this" },+ { type: :tool_call, id: tool_id, name: "echo", content: "{}" },+ { type: :tool, id: tool_id, name: "echo", content: '"result"' },+ ],+ )++ translated = described_class.new(prompt, model).translate++ expect(translated.second[:content]).to start_with("admin: ")+ expect(translated.map { |message| message[:content] }.join).not_to include(tool_id)+ end+it "includes system and user messages separately" dosystem_msg = "This is a system message"user_msg = "user message"diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rbindex d6b22f648ac..677cb495be6 100644--- a/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb@@ -13,6 +13,27 @@ RSpec.describe DiscourseAi::Completions::Dialects::Vllm do)end+ it "embeds participant names in user message content" do+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "System instructions",+ messages: [+ { type: :user, id: "user1", content: "First question" },+ { type: :model, content: "First answer" },+ { type: :user, id: "admin", content: "Who am I?" },+ ],+ )++ expect(described_class.new(prompt, model).translate).to eq(+ [+ { role: "system", content: "System instructions" },+ { role: "user", content: "user1: First question" },+ { role: "assistant", content: "First answer" },+ { role: "user", content: "admin: Who am I?" },+ ],+ )+ end+it "replays tool-call reasoning without adding reasoning to normal assistant messages" doprompt =DiscourseAi::Completions::Prompt.new(