A platform for community discussion. Free, open, simple.
FIX: Improve vLLM completion handling (#41862)
Samc9145ca98c69
diff --git a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rbindex a5aaff1b80e..e3233b2433f 100644--- a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb+++ b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb@@ -272,6 +272,7 @@ module DiscourseAiid: call.id,name: call.name,content: { error: "Not executed — token budget exhausted." }.to_json,+ provider_data: call.provider_data,)end@@ -381,14 +382,21 @@ module DiscourseAicontent = content.to_json if !content.is_a?(String)+ provider_data = deep_symbolize(tool_call["provider_data"])@prompt.push(type: :tool_call,id: id,name: tool_call["name"],content: { arguments: tool_call["parameters"] || {} }.to_json,- provider_data: tool_call["provider_data"],+ provider_data: provider_data,+ )+ @prompt.push(+ type: :tool,+ id: id,+ name: tool_call["name"],+ content: content,+ provider_data: provider_data,)- @prompt.push(type: :tool, id: id, name: tool_call["name"], content: content)endenddiff --git a/plugins/discourse-ai/lib/completions/dialects/dialect.rb b/plugins/discourse-ai/lib/completions/dialects/dialect.rbindex f5919608969..3285c9b586d 100644--- a/plugins/discourse-ai/lib/completions/dialects/dialect.rb+++ b/plugins/discourse-ai/lib/completions/dialects/dialect.rb@@ -20,6 +20,7 @@ module DiscourseAiDiscourseAi::Completions::Dialects::Ollama,DiscourseAi::Completions::Dialects::Mistral,DiscourseAi::Completions::Dialects::Nova,+ DiscourseAi::Completions::Dialects::Vllm,DiscourseAi::Completions::Dialects::OpenAiCompatible,]enddiff --git a/plugins/discourse-ai/lib/completions/dialects/vllm.rb b/plugins/discourse-ai/lib/completions/dialects/vllm.rbnew file mode 100644index 00000000000..ff7f7cf88ed--- /dev/null+++ b/plugins/discourse-ai/lib/completions/dialects/vllm.rb@@ -0,0 +1,74 @@+# frozen_string_literal: true++module DiscourseAi+ module Completions+ module Dialects+ class Vllm < OpenAiCompatible+ class << self+ def can_translate?(llm_model)+ llm_model.provider == "vllm"+ end+ end++ def translate+ merge_tool_batches(super)+ end++ private++ def tool_call_msg(msg)+ translated = super+ if native_tool_support?+ translated[:reasoning_content] = msg[:thinking] if msg[:thinking].present?+ add_tool_batch(translated, msg)+ else+ translated+ end+ end++ def tool_msg(msg)+ return super unless native_tool_support?++ add_tool_batch(super, msg)+ end++ def add_tool_batch(translated, msg)+ batch_id = msg.dig(:provider_data, :vllm, :tool_batch_id)+ translated[:tool_batch_id] = batch_id if batch_id+ translated+ end++ def merge_tool_batches(messages)+ merged = []+ active_batch_id = nil+ active_assistant = nil++ messages.each do |message|+ batch_id = message.delete(:tool_batch_id)++ if batch_id && batch_id == active_batch_id && message[:role] == "assistant" &&+ message[:tool_calls]+ active_assistant[:tool_calls].concat(message[:tool_calls])+ if active_assistant[:reasoning_content].blank? && message[:reasoning_content].present?+ active_assistant[:reasoning_content] = message[:reasoning_content]+ end+ next+ end++ if batch_id && message[:role] == "assistant" && message[:tool_calls]+ active_batch_id = batch_id+ active_assistant = message+ elsif batch_id != active_batch_id || message[:role] != "tool"+ active_batch_id = nil+ active_assistant = nil+ end++ merged << message+ end++ merged+ end+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb b/plugins/discourse-ai/lib/completions/endpoints/vllm.rbindex 7bd58f2bba9..22ff8878fb6 100644--- a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb+++ b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb@@ -28,6 +28,7 @@ module DiscourseAidef decode(response_raw)parsed = JSON.parse(response_raw, symbolize_names: true)result = processor.process_message(parsed)+ attach_tool_batch(result, parsed[:id])if output_thinkingreasoning = parsed.dig(:choices, 0, :message, :reasoning).presence@@ -43,6 +44,8 @@ module DiscourseAielements = [](@decoder << chunk).each do |parsed_json|+ @tool_batch_id = parsed_json[:id] if parsed_json[:id].present?+if output_thinkingreasoning = parsed_json.dig(:choices, 0, :delta, :reasoning).presencereasoning ||= parsed_json.dig(:choices, 0, :delta, :reasoning_content)@@ -61,6 +64,7 @@ module DiscourseAiendresult = processor.process_streamed_message(parsed_json)+ attach_tool_batch(result, parsed_json[:id].presence || @tool_batch_id)elements << result if resultend@@ -78,6 +82,9 @@ module DiscourseAi@thinking = nilendresult.concat(processor.finish)+ attach_tool_batch(result, @tool_batch_id)+ @tool_batch_id = nil+ resultenddef resolve_thinking_config(model_params)@@ -111,6 +118,16 @@ module DiscourseAiprivate+ def attach_tool_batch(result, response_id)+ return if response_id.blank?++ Array(result).each do |item|+ next unless item.is_a?(ToolCall)++ item.provider_data = item.provider_data.deep_merge(vllm: { tool_batch_id: response_id })+ end+ end+def prepare_payload(prompt, model_params, dialect)payload = superdiff --git a/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb b/plugins/discourse-ai/lib/completions/open_ai_message_processor.rbindex 7dc6083d8a9..376dfa6afb4 100644--- a/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb+++ b/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb@@ -127,7 +127,7 @@ module DiscourseAi::Completionsreturn if !usagetoken_details = usage[:prompt_tokens_details] || {}- cached_tokens = token_details[:cached_tokens].to_i+ cached_tokens = (token_details[:cached_tokens] || usage[:prompt_cache_hit_tokens]).to_icache_write_tokens = token_details[:cache_write_tokens].to_iprompt_tokens = usage[:prompt_tokens].to_idiff --git a/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rbnew file mode 100644index 00000000000..d6b22f648ac--- /dev/null+++ b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb@@ -0,0 +1,234 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Completions::Dialects::Vllm do+ fab!(:model, :vllm_model)+ fab!(:open_ai_model, :llm_model)++ before { enable_current_plugin }++ it "is selected only for vLLM models" do+ expect(DiscourseAi::Completions::Dialects::Dialect.dialect_for(model)).to eq(described_class)+ expect(DiscourseAi::Completions::Dialects::Dialect.dialect_for(open_ai_model)).to eq(+ DiscourseAi::Completions::Dialects::ChatGpt,+ )+ end++ it "replays tool-call reasoning without adding reasoning to normal assistant messages" do+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "System instructions",+ messages: [+ { type: :user, content: "First request" },+ {+ type: :tool_call,+ id: "old-tool",+ name: "echo",+ content: { arguments: { text: "old" } }.to_json,+ thinking: "Old tool reasoning",+ },+ { type: :tool, id: "old-tool", name: "echo", content: '"old"' },+ { type: :model, content: "First answer", thinking: "Old answer reasoning" },+ { type: :user, content: "Second request" },+ {+ type: :tool_call,+ id: "current-tool",+ name: "echo",+ content: { arguments: { text: "current" } }.to_json,+ thinking: "Current tool reasoning",+ },+ { type: :tool, id: "current-tool", name: "echo", content: '"current"' },+ ],+ )++ translated = described_class.new(prompt, model).translate+ assistant_tool_calls = translated.select { |message| message[:tool_calls].present? }+ historical_answer = translated.find { |message| message[:content] == "First answer" }++ expect(assistant_tool_calls).to eq(+ [+ {+ role: "assistant",+ content: nil,+ tool_calls: [+ {+ type: "function",+ function: {+ arguments: '{"text":"old"}',+ name: "echo",+ },+ id: "old-tool",+ },+ ],+ reasoning_content: "Old tool reasoning",+ },+ {+ role: "assistant",+ content: nil,+ tool_calls: [+ {+ type: "function",+ function: {+ arguments: '{"text":"current"}',+ name: "echo",+ },+ id: "current-tool",+ },+ ],+ reasoning_content: "Current tool reasoning",+ },+ ],+ )+ expect(historical_answer).to eq(role: "assistant", content: "First answer")+ end++ it "reconstructs parallel tool-call batches" do+ provider_data = { vllm: { tool_batch_id: "response-1" } }+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "System instructions",+ messages: [+ { type: :user, content: "Run both tools" },+ {+ type: :tool_call,+ id: "tool-1",+ name: "echo",+ content: { arguments: { text: "one" } }.to_json,+ thinking: "Call both tools",+ provider_data: provider_data,+ },+ {+ type: :tool,+ id: "tool-1",+ name: "echo",+ content: '"one"',+ provider_data: provider_data,+ },+ {+ type: :tool_call,+ id: "tool-2",+ name: "echo",+ content: { arguments: { text: "two" } }.to_json,+ provider_data: provider_data,+ },+ {+ type: :tool,+ id: "tool-2",+ name: "echo",+ content: '"two"',+ provider_data: provider_data,+ },+ ],+ )++ expect(described_class.new(prompt, model).translate).to eq(+ [+ { role: "system", content: "System instructions" },+ { role: "user", content: "Run both tools" },+ {+ role: "assistant",+ content: nil,+ tool_calls: [+ {+ type: "function",+ function: {+ arguments: '{"text":"one"}',+ name: "echo",+ },+ id: "tool-1",+ },+ {+ type: "function",+ function: {+ arguments: '{"text":"two"}',+ name: "echo",+ },+ id: "tool-2",+ },+ ],+ reasoning_content: "Call both tools",+ },+ { role: "tool", tool_call_id: "tool-1", content: '"one"', name: "echo" },+ { role: "tool", tool_call_id: "tool-2", content: '"two"', name: "echo" },+ ],+ )+ end++ it "does not merge a reused batch ID across user turns" do+ provider_data = { vllm: { tool_batch_id: "reused-response-id" } }+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "System instructions",+ messages: [+ { type: :user, content: "First request" },+ {+ type: :tool_call,+ id: "tool-1",+ name: "echo",+ content: { arguments: { text: "one" } }.to_json,+ provider_data: provider_data,+ },+ {+ type: :tool,+ id: "tool-1",+ name: "echo",+ content: '"one"',+ provider_data: provider_data,+ },+ { type: :model, content: "First answer" },+ { type: :user, content: "Second request" },+ {+ type: :tool_call,+ id: "tool-2",+ name: "echo",+ content: { arguments: { text: "two" } }.to_json,+ provider_data: provider_data,+ },+ {+ type: :tool,+ id: "tool-2",+ name: "echo",+ content: '"two"',+ provider_data: provider_data,+ },+ ],+ )++ assistant_tool_messages =+ described_class.new(prompt, model).translate.select { |message| message[:tool_calls] }++ expect(+ assistant_tool_messages.map { |message| message[:tool_calls].map { |call| call[:id] } },+ ).to eq([%w[tool-1], %w[tool-2]])+ end++ it "keeps vLLM metadata out of XML tool messages" do+ model.update!(provider_params: { disable_native_tools: true })+ provider_data = { vllm: { tool_batch_id: "response-1" } }+ prompt =+ DiscourseAi::Completions::Prompt.new(+ "System instructions",+ messages: [+ { type: :user, content: "Run the tool" },+ {+ type: :tool_call,+ id: "tool-1",+ name: "echo",+ content: { arguments: { text: "one" } }.to_json,+ thinking: "Call the tool",+ provider_data: provider_data,+ },+ {+ type: :tool,+ id: "tool-1",+ name: "echo",+ content: '"one"',+ provider_data: provider_data,+ },+ ],+ )++ translated = described_class.new(prompt, model).translate++ expect(translated.flat_map(&:keys)).not_to include(:reasoning_content, :tool_batch_id)+ end+enddiff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rbindex ff1b924fe4f..e65d8c3ca43 100644--- a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb@@ -1007,6 +1007,27 @@ TEXTendend+ it "uses proper token accounting with top-level cache usage" do+ response = <<~TEXT.strip+ data: {"id":"chatcmpl-cached","object":"chat.completion.chunk","created":1715644203,"model":"gpt-4o-2024-05-13","choices":[],"usage":{"prompt_tokens":25,"completion_tokens":10,"total_tokens":35,"prompt_cache_hit_tokens":5,"prompt_cache_miss_tokens":20}}|+ data: [DONE]+ TEXT++ chunks = response.split("|")+ open_ai_mock.with_chunk_array_support do+ open_ai_mock.stub_raw(chunks)++ dialect = compliance.dialect(prompt: compliance.generic_prompt)+ endpoint.perform_completion!(dialect, user) { |partial| }++ log = AiApiAuditLog.order("id desc").first++ expect(log.request_tokens).to eq(20)+ expect(log.cache_read_tokens).to eq(5)+ expect(log.response_tokens).to eq(10)+ end+ end+it "properly handles multiple params in partial tool calls" do# this is not working and it is driving me nuts so I will use a sledghammer# text = plugin_file_from_fixtures("openai_artifact_call.txt", "bot")diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rbindex 6250f5d1f0f..0375eb24e8d 100644--- a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb@@ -3,6 +3,14 @@require_relative "endpoint_compliance"class VllmMock < EndpointMock+ def response_id+ "cmpl-6sZfAb30Rnv9Q7ufzFwvQsMpjZh8S"+ end++ def invocation_response+ super.tap { |tool_call| tool_call.provider_data = { vllm: { tool_batch_id: response_id } } }+ end+def response(content, tool_call: false)message_content =if tool_call@@ -12,7 +20,7 @@ class VllmMock < EndpointMockend{- id: "cmpl-6sZfAb30Rnv9Q7ufzFwvQsMpjZh8S",+ id: response_id,object: "chat.completion",created: 1_678_464_820,model: "mistralai/Mixtral-8x7B-Instruct-v0.1",@@ -43,7 +51,7 @@ class VllmMock < EndpointMockend+"data: " << {- id: "cmpl-#{SecureRandom.hex}",+ id: response_id,object: "chat.completion.chunk",created: 1_681_283_881,model: "mistralai/Mixtral-8x7B-Instruct-v0.1",@@ -662,6 +670,42 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Vllm doexpect(result[1]).to eq("The answer is 4.")end+ it "assigns one provider batch to parallel tool calls" do+ body = {+ id: "chatcmpl-tool-batch",+ choices: [+ {+ message: {+ role: "assistant",+ content: nil,+ reasoning_content: "I need both results.",+ tool_calls: [+ { id: "tool-1", function: { name: "echo", arguments: '{"text":"one"}' } },+ { id: "tool-2", function: { name: "echo", arguments: '{"text":"two"}' } },+ ],+ },+ },+ ],+ usage: {+ prompt_tokens: 10,+ completion_tokens: 20,+ },+ }++ stub_request(:post, "https://test.dev/v1/chat/completions").to_return(+ status: 200,+ body: body.to_json,+ )++ result = llm.generate("use both tools", user: Discourse.system_user, output_thinking: true)+ tool_calls = result.grep(DiscourseAi::Completions::ToolCall)++ expect(tool_calls.map(&:id)).to eq(%w[tool-1 tool-2])+ expect(tool_calls.map(&:provider_data)).to all(+ eq(vllm: { tool_batch_id: "chatcmpl-tool-batch" }),+ )+ end+it "omits Thinking when output_thinking is false" dobody = {choices: [diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rbindex 9033767f78b..d473d45c74d 100644--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb@@ -54,6 +54,49 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession doeventsend+ describe "custom tool resume" do+ it "reconstructs resumed parallel vLLM tool calls as one assistant batch" do+ vllm_model = Fabricate(:vllm_model)+ ai_agent.update!(default_llm_id: vllm_model.id)+ provider_data = { vllm: { tool_batch_id: "response-1" } }+ tool_calls =+ %w[one two].map do |input|+ DiscourseAi::Completions::ToolCall.new(+ name: "client_tool",+ parameters: {+ input: input,+ },+ id: "tool_#{input}",+ provider_data: provider_data,+ )+ end++ DiscourseAi::Completions::Llm.with_prepared_responses(+ [tool_calls, "Final answer after tools.", "Test title"],+ ) do |_, _, prompts|+ tool_event = collect_events(build_session).find { |type, _| type == :tool_calls }++ collect_events(+ build_session(+ resume_token: tool_event[1][:resume_token],+ tool_results: [+ { tool_call_id: "tool_one", content: "first result" },+ { tool_call_id: "tool_two", content: "second result" },+ ],+ ),+ )++ translated =+ DiscourseAi::Completions::Dialects::Vllm.new(prompts.second, vllm_model).translate+ assistant_tool_messages = translated.select { |message| message[:tool_calls] }++ expect(+ assistant_tool_messages.map { |message| message[:tool_calls].map { |call| call[:id] } },+ ).to eq([%w[tool_one tool_two]])+ end+ end+ end+describe "token budget enforcement" doit "enforces budget before generate when resuming over budget" dotool_call =