Skip to content

A platform for community discussion. Free, open, simple.

FIX: Improve vLLM completion handling (#41862)

Samc9145ca98c69
diff --git a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
index a5aaff1b80e..e3233b2433f 100644
--- a/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
+++ b/plugins/discourse-ai/lib/ai_bot/stream_reply_custom_tools_session.rb
@@ -272,6 +272,7 @@ module DiscourseAi
id: call.id,
name: call.name,
content: { error: "Not executed — token budget exhausted." }.to_json,
+ provider_data: call.provider_data,
)
end
@@ -381,14 +382,21 @@ module DiscourseAi
content = content.to_json if !content.is_a?(String)
+ provider_data = deep_symbolize(tool_call["provider_data"])
@prompt.push(
type: :tool_call,
id: id,
name: tool_call["name"],
content: { arguments: tool_call["parameters"] || {} }.to_json,
- provider_data: tool_call["provider_data"],
+ provider_data: provider_data,
+ )
+ @prompt.push(
+ type: :tool,
+ id: id,
+ name: tool_call["name"],
+ content: content,
+ provider_data: provider_data,
)
- @prompt.push(type: :tool, id: id, name: tool_call["name"], content: content)
end
end
diff --git a/plugins/discourse-ai/lib/completions/dialects/dialect.rb b/plugins/discourse-ai/lib/completions/dialects/dialect.rb
index f5919608969..3285c9b586d 100644
--- a/plugins/discourse-ai/lib/completions/dialects/dialect.rb
+++ b/plugins/discourse-ai/lib/completions/dialects/dialect.rb
@@ -20,6 +20,7 @@ module DiscourseAi
DiscourseAi::Completions::Dialects::Ollama,
DiscourseAi::Completions::Dialects::Mistral,
DiscourseAi::Completions::Dialects::Nova,
+ DiscourseAi::Completions::Dialects::Vllm,
DiscourseAi::Completions::Dialects::OpenAiCompatible,
]
end
diff --git a/plugins/discourse-ai/lib/completions/dialects/vllm.rb b/plugins/discourse-ai/lib/completions/dialects/vllm.rb
new file mode 100644
index 00000000000..ff7f7cf88ed
--- /dev/null
+++ b/plugins/discourse-ai/lib/completions/dialects/vllm.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Completions
+ module Dialects
+ class Vllm < OpenAiCompatible
+ class << self
+ def can_translate?(llm_model)
+ llm_model.provider == "vllm"
+ end
+ end
+
+ def translate
+ merge_tool_batches(super)
+ end
+
+ private
+
+ def tool_call_msg(msg)
+ translated = super
+ if native_tool_support?
+ translated[:reasoning_content] = msg[:thinking] if msg[:thinking].present?
+ add_tool_batch(translated, msg)
+ else
+ translated
+ end
+ end
+
+ def tool_msg(msg)
+ return super unless native_tool_support?
+
+ add_tool_batch(super, msg)
+ end
+
+ def add_tool_batch(translated, msg)
+ batch_id = msg.dig(:provider_data, :vllm, :tool_batch_id)
+ translated[:tool_batch_id] = batch_id if batch_id
+ translated
+ end
+
+ def merge_tool_batches(messages)
+ merged = []
+ active_batch_id = nil
+ active_assistant = nil
+
+ messages.each do |message|
+ batch_id = message.delete(:tool_batch_id)
+
+ if batch_id && batch_id == active_batch_id && message[:role] == "assistant" &&
+ message[:tool_calls]
+ active_assistant[:tool_calls].concat(message[:tool_calls])
+ if active_assistant[:reasoning_content].blank? && message[:reasoning_content].present?
+ active_assistant[:reasoning_content] = message[:reasoning_content]
+ end
+ next
+ end
+
+ if batch_id && message[:role] == "assistant" && message[:tool_calls]
+ active_batch_id = batch_id
+ active_assistant = message
+ elsif batch_id != active_batch_id || message[:role] != "tool"
+ active_batch_id = nil
+ active_assistant = nil
+ end
+
+ merged << message
+ end
+
+ merged
+ end
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
index 7bd58f2bba9..22ff8878fb6 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb
@@ -28,6 +28,7 @@ module DiscourseAi
def decode(response_raw)
parsed = JSON.parse(response_raw, symbolize_names: true)
result = processor.process_message(parsed)
+ attach_tool_batch(result, parsed[:id])
if output_thinking
reasoning = parsed.dig(:choices, 0, :message, :reasoning).presence
@@ -43,6 +44,8 @@ module DiscourseAi
elements = []
(@decoder << chunk).each do |parsed_json|
+ @tool_batch_id = parsed_json[:id] if parsed_json[:id].present?
+
if output_thinking
reasoning = parsed_json.dig(:choices, 0, :delta, :reasoning).presence
reasoning ||= parsed_json.dig(:choices, 0, :delta, :reasoning_content)
@@ -61,6 +64,7 @@ module DiscourseAi
end
result = processor.process_streamed_message(parsed_json)
+ attach_tool_batch(result, parsed_json[:id].presence || @tool_batch_id)
elements << result if result
end
@@ -78,6 +82,9 @@ module DiscourseAi
@thinking = nil
end
result.concat(processor.finish)
+ attach_tool_batch(result, @tool_batch_id)
+ @tool_batch_id = nil
+ result
end
def resolve_thinking_config(model_params)
@@ -111,6 +118,16 @@ module DiscourseAi
private
+ def attach_tool_batch(result, response_id)
+ return if response_id.blank?
+
+ Array(result).each do |item|
+ next unless item.is_a?(ToolCall)
+
+ item.provider_data = item.provider_data.deep_merge(vllm: { tool_batch_id: response_id })
+ end
+ end
+
def prepare_payload(prompt, model_params, dialect)
payload = super
diff --git a/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb b/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb
index 7dc6083d8a9..376dfa6afb4 100644
--- a/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb
+++ b/plugins/discourse-ai/lib/completions/open_ai_message_processor.rb
@@ -127,7 +127,7 @@ module DiscourseAi::Completions
return if !usage
token_details = usage[:prompt_tokens_details] || {}
- cached_tokens = token_details[:cached_tokens].to_i
+ cached_tokens = (token_details[:cached_tokens] || usage[:prompt_cache_hit_tokens]).to_i
cache_write_tokens = token_details[:cache_write_tokens].to_i
prompt_tokens = usage[:prompt_tokens].to_i
diff --git a/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb
new file mode 100644
index 00000000000..d6b22f648ac
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/completions/dialects/vllm_spec.rb
@@ -0,0 +1,234 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Completions::Dialects::Vllm do
+ fab!(:model, :vllm_model)
+ fab!(:open_ai_model, :llm_model)
+
+ before { enable_current_plugin }
+
+ it "is selected only for vLLM models" do
+ expect(DiscourseAi::Completions::Dialects::Dialect.dialect_for(model)).to eq(described_class)
+ expect(DiscourseAi::Completions::Dialects::Dialect.dialect_for(open_ai_model)).to eq(
+ DiscourseAi::Completions::Dialects::ChatGpt,
+ )
+ end
+
+ it "replays tool-call reasoning without adding reasoning to normal assistant messages" do
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System instructions",
+ messages: [
+ { type: :user, content: "First request" },
+ {
+ type: :tool_call,
+ id: "old-tool",
+ name: "echo",
+ content: { arguments: { text: "old" } }.to_json,
+ thinking: "Old tool reasoning",
+ },
+ { type: :tool, id: "old-tool", name: "echo", content: '"old"' },
+ { type: :model, content: "First answer", thinking: "Old answer reasoning" },
+ { type: :user, content: "Second request" },
+ {
+ type: :tool_call,
+ id: "current-tool",
+ name: "echo",
+ content: { arguments: { text: "current" } }.to_json,
+ thinking: "Current tool reasoning",
+ },
+ { type: :tool, id: "current-tool", name: "echo", content: '"current"' },
+ ],
+ )
+
+ translated = described_class.new(prompt, model).translate
+ assistant_tool_calls = translated.select { |message| message[:tool_calls].present? }
+ historical_answer = translated.find { |message| message[:content] == "First answer" }
+
+ expect(assistant_tool_calls).to eq(
+ [
+ {
+ role: "assistant",
+ content: nil,
+ tool_calls: [
+ {
+ type: "function",
+ function: {
+ arguments: '{"text":"old"}',
+ name: "echo",
+ },
+ id: "old-tool",
+ },
+ ],
+ reasoning_content: "Old tool reasoning",
+ },
+ {
+ role: "assistant",
+ content: nil,
+ tool_calls: [
+ {
+ type: "function",
+ function: {
+ arguments: '{"text":"current"}',
+ name: "echo",
+ },
+ id: "current-tool",
+ },
+ ],
+ reasoning_content: "Current tool reasoning",
+ },
+ ],
+ )
+ expect(historical_answer).to eq(role: "assistant", content: "First answer")
+ end
+
+ it "reconstructs parallel tool-call batches" do
+ provider_data = { vllm: { tool_batch_id: "response-1" } }
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System instructions",
+ messages: [
+ { type: :user, content: "Run both tools" },
+ {
+ type: :tool_call,
+ id: "tool-1",
+ name: "echo",
+ content: { arguments: { text: "one" } }.to_json,
+ thinking: "Call both tools",
+ provider_data: provider_data,
+ },
+ {
+ type: :tool,
+ id: "tool-1",
+ name: "echo",
+ content: '"one"',
+ provider_data: provider_data,
+ },
+ {
+ type: :tool_call,
+ id: "tool-2",
+ name: "echo",
+ content: { arguments: { text: "two" } }.to_json,
+ provider_data: provider_data,
+ },
+ {
+ type: :tool,
+ id: "tool-2",
+ name: "echo",
+ content: '"two"',
+ provider_data: provider_data,
+ },
+ ],
+ )
+
+ expect(described_class.new(prompt, model).translate).to eq(
+ [
+ { role: "system", content: "System instructions" },
+ { role: "user", content: "Run both tools" },
+ {
+ role: "assistant",
+ content: nil,
+ tool_calls: [
+ {
+ type: "function",
+ function: {
+ arguments: '{"text":"one"}',
+ name: "echo",
+ },
+ id: "tool-1",
+ },
+ {
+ type: "function",
+ function: {
+ arguments: '{"text":"two"}',
+ name: "echo",
+ },
+ id: "tool-2",
+ },
+ ],
+ reasoning_content: "Call both tools",
+ },
+ { role: "tool", tool_call_id: "tool-1", content: '"one"', name: "echo" },
+ { role: "tool", tool_call_id: "tool-2", content: '"two"', name: "echo" },
+ ],
+ )
+ end
+
+ it "does not merge a reused batch ID across user turns" do
+ provider_data = { vllm: { tool_batch_id: "reused-response-id" } }
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System instructions",
+ messages: [
+ { type: :user, content: "First request" },
+ {
+ type: :tool_call,
+ id: "tool-1",
+ name: "echo",
+ content: { arguments: { text: "one" } }.to_json,
+ provider_data: provider_data,
+ },
+ {
+ type: :tool,
+ id: "tool-1",
+ name: "echo",
+ content: '"one"',
+ provider_data: provider_data,
+ },
+ { type: :model, content: "First answer" },
+ { type: :user, content: "Second request" },
+ {
+ type: :tool_call,
+ id: "tool-2",
+ name: "echo",
+ content: { arguments: { text: "two" } }.to_json,
+ provider_data: provider_data,
+ },
+ {
+ type: :tool,
+ id: "tool-2",
+ name: "echo",
+ content: '"two"',
+ provider_data: provider_data,
+ },
+ ],
+ )
+
+ assistant_tool_messages =
+ described_class.new(prompt, model).translate.select { |message| message[:tool_calls] }
+
+ expect(
+ assistant_tool_messages.map { |message| message[:tool_calls].map { |call| call[:id] } },
+ ).to eq([%w[tool-1], %w[tool-2]])
+ end
+
+ it "keeps vLLM metadata out of XML tool messages" do
+ model.update!(provider_params: { disable_native_tools: true })
+ provider_data = { vllm: { tool_batch_id: "response-1" } }
+ prompt =
+ DiscourseAi::Completions::Prompt.new(
+ "System instructions",
+ messages: [
+ { type: :user, content: "Run the tool" },
+ {
+ type: :tool_call,
+ id: "tool-1",
+ name: "echo",
+ content: { arguments: { text: "one" } }.to_json,
+ thinking: "Call the tool",
+ provider_data: provider_data,
+ },
+ {
+ type: :tool,
+ id: "tool-1",
+ name: "echo",
+ content: '"one"',
+ provider_data: provider_data,
+ },
+ ],
+ )
+
+ translated = described_class.new(prompt, model).translate
+
+ expect(translated.flat_map(&:keys)).not_to include(:reasoning_content, :tool_batch_id)
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
index ff1b924fe4f..e65d8c3ca43 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/open_ai_spec.rb
@@ -1007,6 +1007,27 @@ TEXT
end
end
+ it "uses proper token accounting with top-level cache usage" do
+ response = <<~TEXT.strip
+ data: {"id":"chatcmpl-cached","object":"chat.completion.chunk","created":1715644203,"model":"gpt-4o-2024-05-13","choices":[],"usage":{"prompt_tokens":25,"completion_tokens":10,"total_tokens":35,"prompt_cache_hit_tokens":5,"prompt_cache_miss_tokens":20}}|
+ data: [DONE]
+ TEXT
+
+ chunks = response.split("|")
+ open_ai_mock.with_chunk_array_support do
+ open_ai_mock.stub_raw(chunks)
+
+ dialect = compliance.dialect(prompt: compliance.generic_prompt)
+ endpoint.perform_completion!(dialect, user) { |partial| }
+
+ log = AiApiAuditLog.order("id desc").first
+
+ expect(log.request_tokens).to eq(20)
+ expect(log.cache_read_tokens).to eq(5)
+ expect(log.response_tokens).to eq(10)
+ end
+ end
+
it "properly handles multiple params in partial tool calls" do
# this is not working and it is driving me nuts so I will use a sledghammer
# text = plugin_file_from_fixtures("openai_artifact_call.txt", "bot")
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
index 6250f5d1f0f..0375eb24e8d 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb
@@ -3,6 +3,14 @@
require_relative "endpoint_compliance"
class VllmMock < EndpointMock
+ def response_id
+ "cmpl-6sZfAb30Rnv9Q7ufzFwvQsMpjZh8S"
+ end
+
+ def invocation_response
+ super.tap { |tool_call| tool_call.provider_data = { vllm: { tool_batch_id: response_id } } }
+ end
+
def response(content, tool_call: false)
message_content =
if tool_call
@@ -12,7 +20,7 @@ class VllmMock < EndpointMock
end
{
- id: "cmpl-6sZfAb30Rnv9Q7ufzFwvQsMpjZh8S",
+ id: response_id,
object: "chat.completion",
created: 1_678_464_820,
model: "mistralai/Mixtral-8x7B-Instruct-v0.1",
@@ -43,7 +51,7 @@ class VllmMock < EndpointMock
end
+"data: " << {
- id: "cmpl-#{SecureRandom.hex}",
+ id: response_id,
object: "chat.completion.chunk",
created: 1_681_283_881,
model: "mistralai/Mixtral-8x7B-Instruct-v0.1",
@@ -662,6 +670,42 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Vllm do
expect(result[1]).to eq("The answer is 4.")
end
+ it "assigns one provider batch to parallel tool calls" do
+ body = {
+ id: "chatcmpl-tool-batch",
+ choices: [
+ {
+ message: {
+ role: "assistant",
+ content: nil,
+ reasoning_content: "I need both results.",
+ tool_calls: [
+ { id: "tool-1", function: { name: "echo", arguments: '{"text":"one"}' } },
+ { id: "tool-2", function: { name: "echo", arguments: '{"text":"two"}' } },
+ ],
+ },
+ },
+ ],
+ usage: {
+ prompt_tokens: 10,
+ completion_tokens: 20,
+ },
+ }
+
+ stub_request(:post, "https://test.dev/v1/chat/completions").to_return(
+ status: 200,
+ body: body.to_json,
+ )
+
+ result = llm.generate("use both tools", user: Discourse.system_user, output_thinking: true)
+ tool_calls = result.grep(DiscourseAi::Completions::ToolCall)
+
+ expect(tool_calls.map(&:id)).to eq(%w[tool-1 tool-2])
+ expect(tool_calls.map(&:provider_data)).to all(
+ eq(vllm: { tool_batch_id: "chatcmpl-tool-batch" }),
+ )
+ end
+
it "omits Thinking when output_thinking is false" do
body = {
choices: [
diff --git a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
index 9033767f78b..d473d45c74d 100644
--- a/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/ai_bot/stream_reply_custom_tools_session_spec.rb
@@ -54,6 +54,49 @@ RSpec.describe DiscourseAi::AiBot::StreamReplyCustomToolsSession do
events
end
+ describe "custom tool resume" do
+ it "reconstructs resumed parallel vLLM tool calls as one assistant batch" do
+ vllm_model = Fabricate(:vllm_model)
+ ai_agent.update!(default_llm_id: vllm_model.id)
+ provider_data = { vllm: { tool_batch_id: "response-1" } }
+ tool_calls =
+ %w[one two].map do |input|
+ DiscourseAi::Completions::ToolCall.new(
+ name: "client_tool",
+ parameters: {
+ input: input,
+ },
+ id: "tool_#{input}",
+ provider_data: provider_data,
+ )
+ end
+
+ DiscourseAi::Completions::Llm.with_prepared_responses(
+ [tool_calls, "Final answer after tools.", "Test title"],
+ ) do |_, _, prompts|
+ tool_event = collect_events(build_session).find { |type, _| type == :tool_calls }
+
+ collect_events(
+ build_session(
+ resume_token: tool_event[1][:resume_token],
+ tool_results: [
+ { tool_call_id: "tool_one", content: "first result" },
+ { tool_call_id: "tool_two", content: "second result" },
+ ],
+ ),
+ )
+
+ translated =
+ DiscourseAi::Completions::Dialects::Vllm.new(prompts.second, vllm_model).translate
+ assistant_tool_messages = translated.select { |message| message[:tool_calls] }
+
+ expect(
+ assistant_tool_messages.map { |message| message[:tool_calls].map { |call| call[:id] } },
+ ).to eq([%w[tool_one tool_two]])
+ end
+ end
+ end
+
describe "token budget enforcement" do
it "enforces budget before generate when resuming over budget" do
tool_call =