A platform for community discussion. Free, open, simple.
DEV: Allow plugins to inject request headers for LLM completions (#41089)
Rafael dos Santos Silvabe759044b50e
diff --git a/plugins/discourse-ai/lib/completions/endpoints/base.rb b/plugins/discourse-ai/lib/completions/endpoints/base.rbindex 99f74b8aa46..fd5850fbbd6 100644--- a/plugins/discourse-ai/lib/completions/endpoints/base.rb+++ b/plugins/discourse-ai/lib/completions/endpoints/base.rb@@ -9,6 +9,23 @@ module DiscourseAiCompletionFailed = Class.new(StandardError)FAIL_THRESHOLD = 5FAIL_TTL = 1.hour++ # Read-only context handed to request header providers. Lets plugins+ # contribute provider-agnostic, per-request HTTP headers (e.g. routing+ # hints for a proxy) without reaching into endpoint internals.+ RequestHeaderContext =+ Struct.new(+ :llm_model,+ :feature_name,+ :feature_context,+ :has_images,+ :streaming,+ keyword_init: true,+ )++ # Mutated in place (<<, clear) but never reassigned, so the single array+ # is shared across Base and all endpoint subclasses via constant lookup.+ REQUEST_HEADERS_PROVIDERS = []# 6 minutes# Reasoning LLMs can take a very long time to respond, generally it will be under 5 minutes# The alternative is to have per LLM timeouts but that would make it extra confusing for people@@ -44,6 +61,22 @@ module DiscourseAidef can_contact?(_llm_model)raise NotImplementedErrorend++ # Register a block that returns a Hash of extra HTTP headers to send+ # with each completion request. The block receives a+ # RequestHeaderContext and must not raise (errors are swallowed and+ # logged so a header bug can never break inference).+ def register_request_headers_provider(&block)+ REQUEST_HEADERS_PROVIDERS << block+ end++ def request_headers_providers+ REQUEST_HEADERS_PROVIDERS+ end++ def reset_request_headers_providers!+ REQUEST_HEADERS_PROVIDERS.clear+ endenddef initialize(llm_model)@@ -95,6 +128,8 @@ module DiscourseAi@forced_json_through_prefill = false@partial_tool_calls = partial_tool_calls@output_thinking = output_thinking+ @feature_name = feature_name+ @feature_context = feature_contextmax_tokens = enforce_max_output_tokens(model_params[:max_tokens])model_params[:max_tokens] = max_tokens if max_tokens@@ -127,6 +162,7 @@ module DiscourseAi@streaming_mode = block_given?prompt = dialect.translate+ @request_has_images = prompt_has_images?(prompt)structured_output = nil@@ -396,12 +432,56 @@ module DiscourseAiattr_reader :llm_model+ # Extra HTTP headers contributed by registered providers for this+ # request. Endpoints that want them merge the result into their request+ # headers. Provider failures are isolated so they can never break a+ # completion.+ def extra_request_headers+ providers = self.class.request_headers_providers+ return {} if providers.blank?++ context =+ RequestHeaderContext.new(+ llm_model: llm_model,+ feature_name: @feature_name,+ feature_context: @feature_context,+ has_images: @request_has_images,+ streaming: @streaming_mode,+ )++ providers.each_with_object({}) do |provider, headers|+ result = provider.call(context)+ headers.merge!(result.stringify_keys) if result.is_a?(Hash)+ rescue StandardError => e+ Discourse.warn_exception(+ e,+ message: "Discourse AI request header provider raised an error; skipping it",+ )+ end+ end+protecteddef tokenizerllm_model.tokenizer_classend+ # Detects whether the translated prompt carries image content, so+ # providers can flag vision requests. Mirrors the OpenAI-compatible+ # shape (messages with an array content holding "image_url" parts) used+ # by the providers that consume this; degrades to false otherwise.+ def prompt_has_images?(translated_prompt)+ return false if !translated_prompt.is_a?(Array)++ translated_prompt.any? do |message|+ content = message[:content] if message.is_a?(Hash)+ content.is_a?(Array) &&+ content.any? { |part| part.is_a?(Hash) && part[:type] == "image_url" }+ end+ rescue StandardError+ false+ end+# should normalize temperature, max_tokens, stop_words to endpoint specific valuesdef normalize_model_params(model_params)raise NotImplementedErrordiff --git a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb b/plugins/discourse-ai/lib/completions/endpoints/vllm.rbindex 5f6c738d26d..6654c021857 100644--- a/plugins/discourse-ai/lib/completions/endpoints/vllm.rb+++ b/plugins/discourse-ai/lib/completions/endpoints/vllm.rb@@ -155,6 +155,8 @@ module DiscourseAiapi_key = llm_model&.api_key || SiteSetting.ai_vllm_api_keyheaders["Authorization"] = "Bearer #{api_key}" if api_key.present?+ headers.merge!(extra_request_headers)+Net::HTTP::Post.new(model_uri, headers).tap { |r| r.body = payload }endenddiff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rbindex fbdb7be8387..6250f5d1f0f 100644--- a/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/vllm_spec.rb@@ -750,4 +750,82 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Vllm doendendend++ describe "request header providers" do+ after { described_class.reset_request_headers_providers! }++ def capture_headers(feature_name: nil)+ captured = nil+ stub =+ stub_request(:post, "https://test.dev/v1/chat/completions")+ .with do |request|+ captured = request.headers+ true+ end+ .to_return(status: 200, body: completion_response_body)++ llm.generate("say hello", user: Discourse.system_user, feature_name: feature_name)++ expect(stub).to have_been_requested+ captured+ end++ it "merges headers contributed by a registered provider" do+ described_class.register_request_headers_provider do |context|+ { "X-Test-Feature" => context.feature_name || "default" }+ end++ headers = capture_headers(feature_name: "summarize")++ expect(headers["X-Test-Feature"]).to eq("summarize")+ end++ it "passes request context (llm_model, streaming, vision) to the provider" do+ captured_context = nil+ described_class.register_request_headers_provider do |context|+ captured_context = context+ {}+ end++ capture_headers(feature_name: "ai_helper")++ expect(captured_context.llm_model).to eq(llm_model)+ expect(captured_context.feature_name).to eq("ai_helper")+ expect(captured_context.streaming).to eq(false)+ expect(captured_context.has_images).to eq(false)+ end++ it "isolates provider failures so the completion still succeeds" do+ described_class.register_request_headers_provider { |_context| raise "boom" }+ described_class.register_request_headers_provider { |_context| { "X-Test-Survivor" => "1" } }++ headers = nil+ expect { headers = capture_headers }.not_to raise_error+ expect(headers["X-Test-Survivor"]).to eq("1")+ end++ it "sends no extra headers when nothing is registered" do+ headers = capture_headers+ expect(headers.keys).not_to include("X-Test-Feature")+ end+ end++ describe "#prompt_has_images?" do+ it "is true when a message carries an image_url content part" do+ prompt = [+ { role: "user", content: [{ type: "text", text: "hi" }] },+ {+ role: "user",+ content: [{ type: "image_url", image_url: { url: "data:image/png;base64,abc" } }],+ },+ ]++ expect(endpoint.send(:prompt_has_images?, prompt)).to eq(true)+ end++ it "is false for text-only or non-array prompts" do+ expect(endpoint.send(:prompt_has_images?, [{ role: "user", content: "hi" }])).to eq(false)+ expect(endpoint.send(:prompt_has_images?, nil)).to eq(false)+ end+ endend