Skip to content

A platform for community discussion. Free, open, simple.

FEATURE: retry transient and rate-limited LLM completion requests (#41237)

Sam3d737f966e25
diff --git a/db/structure.sql b/db/structure.sql
index 9ca2acbd86b..88e9935df6a 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -517,7 +517,8 @@ CREATE TABLE public.ai_api_audit_logs (
cache_write_tokens integer,
cache_read_tokens integer,
llm_id bigint,
- response_status integer
+ response_status integer,
+ request_attempts jsonb
);
@@ -22274,6 +22275,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260629022603'),
('20260626055145'),
('20260624140945'),
('20260623090824'),
diff --git a/plugins/discourse-ai/app/models/ai_api_audit_log.rb b/plugins/discourse-ai/app/models/ai_api_audit_log.rb
index 612c66b661e..175df4f4315 100644
--- a/plugins/discourse-ai/app/models/ai_api_audit_log.rb
+++ b/plugins/discourse-ai/app/models/ai_api_audit_log.rb
@@ -1,7 +1,9 @@
# frozen_string_literal: true
class AiApiAuditLog < ActiveRecord::Base
- self.ignored_columns = %w[cached_tokens] # TODO: Remove when 20251118000500_drop_cached_tokens_from_ai_api_audit_logs has been promoted to pre-deploy
+ self.ignored_columns += [
+ "cached_tokens", # TODO: Remove when 20251118000500_drop_cached_tokens_from_ai_api_audit_logs has been promoted to pre-deploy
+ ]
belongs_to :post
belongs_to :topic
belongs_to :user
@@ -64,6 +66,7 @@ end
# language_model :string(255)
# raw_request_payload :string
# raw_response_payload :string
+# request_attempts :jsonb
# request_tokens :integer
# response_status :integer
# response_tokens :integer
diff --git a/plugins/discourse-ai/app/models/llm_model.rb b/plugins/discourse-ai/app/models/llm_model.rb
index 65e1cf159cf..34b1dc5e2e1 100644
--- a/plugins/discourse-ai/app/models/llm_model.rb
+++ b/plugins/discourse-ai/app/models/llm_model.rb
@@ -258,6 +258,11 @@ class LlmModel < ActiveRecord::Base
hidden_if: :enable_thinking,
},
disable_top_p: :checkbox,
+ service_tier: {
+ type: :enum,
+ values: %w[default standard flex priority],
+ default: "default",
+ },
},
azure: {
disable_native_tools: :checkbox,
diff --git a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb
index a4a1c74af8c..33d705eeae5 100644
--- a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb
+++ b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb
@@ -15,6 +15,8 @@ class AiApiAuditLogSerializer < ApplicationSerializer
:feature_name,
:llm_id,
:language_model,
+ :response_status,
+ :request_attempts,
:created_at,
:prev_log_id,
:next_log_id,
@@ -25,6 +27,14 @@ class AiApiAuditLogSerializer < ApplicationSerializer
:conversation_cache_write_tokens,
:conversation_spending
+ def request_attempts
+ object.request_attempts
+ end
+
+ def include_request_attempts?
+ object.has_attribute?(:request_attempts)
+ end
+
def spending
object.llm_model&.spending_for(object)
end
diff --git a/plugins/discourse-ai/db/migrate/20260629022603_add_request_attempts_to_ai_api_audit_logs.rb b/plugins/discourse-ai/db/migrate/20260629022603_add_request_attempts_to_ai_api_audit_logs.rb
new file mode 100644
index 00000000000..3e30398ac09
--- /dev/null
+++ b/plugins/discourse-ai/db/migrate/20260629022603_add_request_attempts_to_ai_api_audit_logs.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class AddRequestAttemptsToAiApiAuditLogs < ActiveRecord::Migration[8.0]
+ def up
+ unless column_exists?(:ai_api_audit_logs, :request_attempts)
+ add_column :ai_api_audit_logs, :request_attempts, :jsonb
+ end
+ end
+
+ def down
+ raise ActiveRecord::IrreversibleMigration
+ end
+end
diff --git a/plugins/discourse-ai/lib/completions/cancel_manager.rb b/plugins/discourse-ai/lib/completions/cancel_manager.rb
index 82ce562b21c..e3e5322db08 100644
--- a/plugins/discourse-ai/lib/completions/cancel_manager.rb
+++ b/plugins/discourse-ai/lib/completions/cancel_manager.rb
@@ -9,6 +9,7 @@ module DiscourseAi
def initialize
@cancelled = false
+ @cancel_event = Concurrent::Event.new
@callbacks = Concurrent::Array.new
@mutex = Mutex.new
@monitor_thread = nil
@@ -79,12 +80,17 @@ module DiscourseAi
@callbacks << cb
end
+ def wait_for_cancel(timeout)
+ @cancel_event.wait(timeout)
+ end
+
def remove_callback(cb)
@callbacks.delete(cb)
end
def cancel!
@cancelled = true
+ @cancel_event.set
monitor_thread = @monitor_thread
if monitor_thread && monitor_thread != Thread.current
monitor_thread.wakeup
diff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
index 5e387c824a2..847dcd8c670 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb
@@ -176,6 +176,7 @@ module DiscourseAi
response_data
end
rescue Aws::BedrockRuntime::Errors::ServiceError => e
+ log.response_status = e.context&.http_response&.status_code if log
Rails.logger.error("#{self.class.name}: #{e.class}: #{e.message}")
raise CompletionFailed, e.message
ensure
@@ -190,7 +191,7 @@ module DiscourseAi
log.cache_write_tokens =
processor.cache_write_input_tokens if processor.cache_write_input_tokens
log.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?
- log.response_status ||= 200
+ log.response_status ||= 200 if call_status == :success
log.created_at = start_time
log.updated_at = Time.now
log.duration_msecs = (Time.now - start_time) * 1000
@@ -277,6 +278,9 @@ module DiscourseAi
region = llm_model.lookup_custom_param("region")
client_options = { region: region, http_read_timeout: TIMEOUT }
+ # AWS SDK already retries modeled throttling, 5xx, and networking failures
+ # by default. Keep Bedrock Converse on SDK-managed retries to avoid
+ # double-retrying request streams in Discourse's HTTP retry loop.
role_arn = llm_model.lookup_custom_param("role_arn")
access_key_id = llm_model.lookup_custom_param("access_key_id")
diff --git a/plugins/discourse-ai/lib/completions/endpoints/base.rb b/plugins/discourse-ai/lib/completions/endpoints/base.rb
index fd5850fbbd6..d9aa0b6dcb7 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/base.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/base.rb
@@ -9,6 +9,24 @@ module DiscourseAi
CompletionFailed = Class.new(StandardError)
FAIL_THRESHOLD = 5
FAIL_TTL = 1.hour
+ # Applies to both synchronous and background completions; keep retry waits bounded
+ # and interruptible via sleep_before_retry.
+ RATE_LIMIT_RETRY_DELAYS = [2, 8, 16]
+ RETRY_JITTER_MAX_SECONDS = 1.0
+ TRANSIENT_ERROR_RETRY_DELAYS = [0.5, 1.0]
+ MAX_RETRY_AFTER_SECONDS = 60
+ # Stored in AiApiAuditLog#request_attempts for retried network failures,
+ # which do not have an HTTP status code. 0 is intentionally outside the HTTP range.
+ NETWORK_ERROR_RETRY_STATUS = 0
+ RETRIABLE_NETWORK_ERRORS = [
+ Net::OpenTimeout,
+ Net::ReadTimeout,
+ Errno::ECONNREFUSED,
+ Errno::ECONNRESET,
+ Errno::ETIMEDOUT,
+ SocketError,
+ OpenSSL::SSL::SSLError,
+ ]
# Read-only context handed to request header providers. Lets plugins
# contribute provider-agnostic, per-request HTTP headers (e.g. routing
@@ -118,8 +136,6 @@ module DiscourseAi
LlmQuota.check_quotas!(@llm_model, user)
LlmCreditAllocation.check_credits!(@llm_model, feature_name)
- start_time = Time.now
-
if cancel_manager && cancel_manager.cancelled?
# nothing to do
return
@@ -134,11 +150,10 @@ module DiscourseAi
max_tokens = enforce_max_output_tokens(model_params[:max_tokens])
model_params[:max_tokens] = max_tokens if max_tokens
model_params = normalize_model_params(model_params)
- orig_blk = blk
if block_given? && disable_streaming?
- result =
- perform_completion!(
+ return(
+ replay_non_streaming_as_streaming!(
dialect,
user,
model_params,
@@ -147,273 +162,314 @@ module DiscourseAi
partial_tool_calls: partial_tool_calls,
output_thinking: output_thinking,
cancel_manager: cancel_manager,
- execution_context:,
+ execution_context: execution_context,
+ &blk
)
-
- wrapped = result
- wrapped = [result] if !result.is_a?(Array)
- wrapped.each do |partial|
- blk.call(partial)
- break if cancel_manager&.cancelled?
- end
- return result
+ )
end
@streaming_mode = block_given?
prompt = dialect.translate
@request_has_images = prompt_has_images?(prompt)
+ perform_completion_request_with_retries(
+ prompt: prompt,
+ dialect: dialect,
+ user: user,
+ model_params: model_params,
+ feature_name: feature_name,
+ feature_context: feature_context,
+ partial_tool_calls: partial_tool_calls,
+ orig_blk: blk,
+ cancel_manager: cancel_manager,
+ execution_context: execution_context,
+ )
+ end
- structured_output = nil
-
- if model_params[:response_format].present?
- schema_properties =
- model_params[:response_format].dig(:json_schema, :schema, :properties)
+ def replay_non_streaming_as_streaming!(
+ dialect,
+ user,
+ model_params,
+ feature_name:,
+ feature_context:,
+ partial_tool_calls:,
+ output_thinking:,
+ cancel_manager:,
+ execution_context:,
+ &blk
+ )
+ result =
+ perform_completion!(
+ dialect,
+ user,
+ model_params,
+ feature_name: feature_name,
+ feature_context: feature_context,
+ partial_tool_calls: partial_tool_calls,
+ output_thinking: output_thinking,
+ cancel_manager: cancel_manager,
+ execution_context: execution_context,
+ )
- if schema_properties.present?
- structured_output = DiscourseAi::Completions::StructuredOutput.new(schema_properties)
- end
+ wrapped = result
+ wrapped = [result] if !result.is_a?(Array)
+ wrapped.each do |partial|
+ blk.call(partial)
+ break if cancel_manager&.cancelled?
end
+ result
+ end
+
+ def build_structured_output(model_params)
+ return if model_params[:response_format].blank?
+
+ schema_properties = model_params[:response_format].dig(:json_schema, :schema, :properties)
+ return if schema_properties.blank?
- cancel_manager_callback = nil
+ DiscourseAi::Completions::StructuredOutput.new(schema_properties)
+ end
+
+ def perform_completion_request_with_retries(
+ prompt:,
+ dialect:,
+ user:,
+ model_params:,
+ feature_name:,
+ feature_context:,
+ partial_tool_calls:,
+ orig_blk:,
+ cancel_manager:,
+ execution_context:
+ )
+ request_started_at = Time.now
cancelled = false
call_status = :error
+ retry_count_429 = 0
+ # 408/409/5xx and network errors share one transient budget.
+ retry_count_transient = 0
+ request_attempts = []
+ retried = false
+ next_attempt_delay_ms = 0
+ @forced_json_through_prefill = false
+ request_body = prepare_payload(prompt, model_params, dialect).to_json
+ log =
+ start_completion_log(
+ request_body: request_body,
+ dialect: dialect,
+ prompt: prompt,
+ user: user,
+ feature_name: feature_name,
+ feature_context: feature_context,
+ )
- FinalDestination::HTTP.start(
- model_uri.host,
- model_uri.port,
- use_ssl: use_ssl?,
- read_timeout: TIMEOUT,
- open_timeout: TIMEOUT,
- write_timeout: TIMEOUT,
- ) do |http|
- if cancel_manager
- cancel_manager_callback =
- lambda do
- cancelled = true
- call_status = :cancelled
- http.finish
- end
- cancel_manager.add_callback(cancel_manager_callback)
- end
+ loop do
+ call_status = :error
response_data = +""
response_raw = +""
# Needed to response token calculations. Cannot rely on response_data due to function buffering.
partials_raw = +""
- request_body = prepare_payload(prompt, model_params, dialect).to_json
+ structured_output = build_structured_output(model_params)
request = prepare_request(request_body)
-
- # Some providers rely on prefill to return structured outputs, so the start
- # of the JSON won't be included in the response. Supply it to keep JSON valid.
- structured_output << +"{" if structured_output && @forced_json_through_prefill
-
- log = nil
- start_logging =
- lambda do
- log =
- start_log(
- provider_id: provider_id,
- request_body: request_body,
- dialect: dialect,
- prompt: prompt,
- user: user,
- feature_name: feature_name,
- feature_context: feature_context,
- )
- end
-
- start_logging.call
+ retrying = false
+ retry_delay = nil
+ retry_status = nil
+ current_attempt_delay_ms = next_attempt_delay_ms
+ next_attempt_delay_ms = 0
+ response_output_started = false
+ cancel_manager_callback = nil
if cancelled || cancel_manager&.cancelled?
call_status = :cancelled
break
end
- http.request(request) do |response|
- log.response_status = response.code.to_i if log
+ begin
+ FinalDestination::HTTP.start(
+ model_uri.host,
+ model_uri.port,
+ use_ssl: use_ssl?,
+ read_timeout: TIMEOUT,
+ open_timeout: TIMEOUT,
+ write_timeout: TIMEOUT,
+ ) do |http|
+ if cancel_manager
+ cancel_manager_callback =
+ lambda do
+ cancelled = true
+ call_status = :cancelled
+ http.finish
+ end
+ cancel_manager.add_callback(cancel_manager_callback)
+ end
- if response.code.to_i != 200
- Rails.logger.error(
- "#{self.class.name}: status: #{response.code.to_i} - body: #{response.body}",
- )
- response_raw << response.body.to_s
- raise CompletionFailed, response.body
- end
+ begin
+ http.request(request) do |response|
+ log.response_status = response.code.to_i if log
+
+ if response.code.to_i != 200
+ retry_status, retry_delay =
+ failed_response_retry_status_and_delay(
+ response,
+ response_raw,
+ retry_count_429: retry_count_429,
+ retry_count_transient: retry_count_transient,
+ )
+ retrying = !retry_delay.nil?
+ raise CompletionFailed, response.body
+ end
+
+ # Some providers rely on prefill to return structured outputs, so the start
+ # of the JSON won't be included in the response. Supply it to keep JSON valid.
+ structured_output << +"{" if structured_output && @forced_json_through_prefill
- xml_tool_processor =
- XmlToolProcessor.new(
- partial_tool_calls: partial_tool_calls,
- tool_definitions: dialect.prompt.tools,
- ) if xml_tools_enabled? && dialect.prompt.has_tools?
-
- to_strip = xml_tags_to_strip(dialect)
- xml_stripper =
- DiscourseAi::Completions::XmlTagStripper.new(to_strip) if to_strip.present?
-
- if @streaming_mode
- blk =
- lambda do |partial|
- if partial.is_a?(String)
- partial = xml_stripper << partial if xml_stripper && !partial.empty?
-
- if structured_output.present?
- structured_output << partial if !partial.empty?
- partial = structured_output
- end
+ xml_tool_processor = build_xml_tool_processor(dialect, partial_tool_calls)
+ xml_stripper = build_xml_stripper(dialect)
+
+ if @streaming_mode
+ blk = streaming_partial_handler(orig_blk, xml_stripper, structured_output)
end
- orig_blk.call(partial) if partial
- end
- end
- if !@streaming_mode
- response_data =
- non_streaming_response(
- response: response,
- xml_tool_processor: xml_tool_processor,
- xml_stripper: xml_stripper,
- partials_raw: partials_raw,
- response_raw: response_raw,
- structured_output: structured_output,
- )
- call_status = :success
- return response_data
- end
+ if !@streaming_mode
+ response_data =
+ non_streaming_response(
+ response: response,
+ xml_tool_processor: xml_tool_processor,
+ xml_stripper: xml_stripper,
+ partials_raw: partials_raw,
+ response_raw: response_raw,
+ structured_output: structured_output,
+ )
+ call_status = :success
+ return response_data
+ end
- response.read_body do |chunk|
- break if cancelled
+ response_data =
+ streaming_response(
+ response: response,
+ blk: blk,
+ xml_tool_processor: xml_tool_processor,
+ xml_stripper: xml_stripper,
+ partials_raw: partials_raw,
+ response_raw: response_raw,
+ structured_output: structured_output,
+ cancelled: -> { cancelled },
+ on_output_started: -> { response_output_started = true },
+ )
+ call_status = :success
+ return response_data
+ end
+ rescue *RETRIABLE_NETWORK_ERRORS => e
+ raise if cancelled
- response_raw << chunk
+ Rails.logger.warn(
+ "#{self.class.name}: retryable network error: #{e.class}: #{e.message}",
+ )
+ if response_output_started
+ retrying = false
+ raise CompletionFailed, e.message
+ end
- decode_chunk(chunk).each do |partial|
- break if cancelled
- partials_raw << partial.to_s
- response_data << partial if partial.is_a?(String)
- partials = [partial]
- if xml_tool_processor && partial.is_a?(String)
- partials = (xml_tool_processor << partial)
- break if xml_tool_processor.should_cancel?
+ retry_status = NETWORK_ERROR_RETRY_STATUS
+ retry_delay =
+ retry_delay_for_network_error(retry_count_transient: retry_count_transient)
+ retrying = !retry_delay.nil?
+ raise CompletionFailed, e.message
+ ensure
+ if cancel_manager && cancel_manager_callback
+ cancel_manager.remove_callback(cancel_manager_callback)
end
- partials.each { |inner_partial| blk.call(inner_partial) }
end
end
+ rescue *RETRIABLE_NETWORK_ERRORS => e
+ raise if cancelled
- if xml_stripper
- stripped = xml_stripper.finish
- if stripped.present?
- response_data << stripped
- result = []
- result = (xml_tool_processor << stripped) if xml_tool_processor
- result.each { |partial| blk.call(partial) }
- end
+ Rails.logger.warn(
+ "#{self.class.name}: retryable network error: #{e.class}: #{e.message}",
+ )
+ retry_status = NETWORK_ERROR_RETRY_STATUS
+ retry_delay =
+ retry_delay_for_network_error(retry_count_transient: retry_count_transient)
+ retrying = !retry_delay.nil?
+
+ if retrying
+ request_attempts << request_attempt(retry_status, current_attempt_delay_ms)
+ retried = true
+ next_attempt_delay_ms = retry_delay_to_ms(retry_delay)
+ retry_count_transient += 1
+ sleep_before_retry(retry_delay, cancel_manager) if retry_delay.positive?
+ next
end
- xml_tool_processor.finish.each { |partial| blk.call(partial) } if xml_tool_processor
- decode_chunk_finish.each { |partial| blk.call(partial) }
-
- if structured_output
- structured_output.finish
- if structured_output.broken?
- # signal last partial output which will get parsed
- # by best effort json parser
- blk.call("")
+
+ raise CompletionFailed, e.message
+ rescue CompletionFailed
+ if retrying && !cancelled
+ if retry_status
+ request_attempts << request_attempt(retry_status, current_attempt_delay_ms)
+ retried = true
+ next_attempt_delay_ms = retry_delay_to_ms(retry_delay)
+ end
+
+ if retry_status == 429
+ retry_count_429 += 1
else
- # got to signal the end of structured output
- blk.call(structured_output)
+ retry_count_transient += 1
end
+
+ sleep_before_retry(retry_delay, cancel_manager) if retry_delay.positive?
+ next
end
- call_status = :success
- return response_data
+
+ raise
ensure
- should_log = log && call_status != :cancelled
+ should_log = log && call_status != :cancelled && !retrying
if should_log
- log.raw_response_payload = response_raw
- final_log_update(log)
- log.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?
- log.response_status ||= 200
- log.created_at = start_time
- log.updated_at = Time.now
- log.duration_msecs = (Time.now - start_time) * 1000
- log.save!
-
- execution_context&.token_usage_tracker&.add_from_audit_log(log)
-
- AiApiRequestStat.record_from_audit_log(log, llm_model: @llm_model)
- LlmQuota.log_usage(@llm_model, user, log.request_tokens, log.response_tokens)
- LlmCreditAllocation.deduct_credits!(
- @llm_model,
- feature_name,
- log.request_tokens,
- log.response_tokens,
- )
+ if retried
+ final_attempt_status = retry_status || log.response_status
+ if final_attempt_status
+ request_attempts << request_attempt(
+ final_attempt_status,
+ current_attempt_delay_ms,
+ )
+ end
+ end
- # Record Prometheus metrics
- DiscourseAi::Completions::LlmMetric.record(
- llm_model: @llm_model,
+ persist_completion_log!(
+ log,
+ response_raw: response_raw,
+ partials_raw: partials_raw,
+ request_attempts: request_attempts.presence,
+ call_status: call_status,
+ start_time: request_started_at,
feature_name: feature_name,
- request_tokens: log.request_tokens || 0,
- response_tokens: log.response_tokens || 0,
- duration_ms: log.duration_msecs,
- status: call_status,
+ user: user,
+ execution_context: execution_context,
)
-
- if Rails.env.development? && ENV["DISCOURSE_AI_DEBUG"]
- puts "#{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"
- end
end
- track_failures(call_status)
-
- if should_log && (logger = execution_context&.audit_logger)
- call_data = <<~LOG
- #{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}
- request:
- #{format_possible_json_payload(log.raw_request_payload)}
- response:
- #{response_data}
- LOG
- logger.info(call_data)
- end
- if should_log && (structured_logger = execution_context&.structured_audit_logger)
- llm_request =
- begin
- JSON.parse(log.raw_request_payload)
- rescue StandardError
- log.raw_request_payload
- end
+ track_failures(call_status) if !retrying
- # gemini puts passwords in query params
- # we don't want to log that
- llm_call_step = structured_logger.add_child_step(name: "Performing LLM call")
-
- structured_logger.append_entry(
- step: llm_call_step,
- name: "llm_call",
- args: {
- class: self.class.name,
- completion_url: request.uri.to_s.split("?")[0],
- request: llm_request,
- result: response_data,
- request_tokens: log.request_tokens,
- response_tokens: log.response_tokens,
- duration: log.duration_msecs,
- stream: @streaming_mode,
- },
- started_at: start_time.utc,
- ended_at: Time.now.utc,
+ if should_log
+ log_completion_audit_entries(
+ log,
+ request: request,
+ response_data: response_data,
+ start_time: request_started_at,
+ execution_context: execution_context,
)
end
end
end
rescue IOError, StandardError
raise if !cancelled
- ensure
- if cancel_manager && cancel_manager_callback
- cancel_manager.remove_callback(cancel_manager_callback)
- end
end
+ private :replay_non_streaming_as_streaming!,
+ :build_structured_output,
+ :perform_completion_request_with_retries
+
def final_log_update(log)
# for people that need to override
end
@@ -525,6 +581,357 @@ module DiscourseAi
private
+ def start_completion_log(
+ request_body:,
+ dialect:,
+ prompt:,
+ user:,
+ feature_name:,
+ feature_context:
+ )
+ start_log(
+ provider_id: provider_id,
+ request_body: request_body,
+ dialect: dialect,
+ prompt: prompt,
+ user: user,
+ feature_name: feature_name,
+ feature_context: feature_context,
+ )
+ end
+
+ # Populated only once retrying occurs. Each entry represents an issued request in
+ # the retried sequence, and delay_ms is the planned wait before that request.
+ def request_attempt(status, delay_ms)
+ { "status" => status, "delay_ms" => delay_ms }
+ end
+
+ def retry_delay_to_ms(retry_delay)
+ (retry_delay.to_f * 1000).round
+ end
+
+ def failed_response_retry_status_and_delay(
+ response,
+ response_raw,
+ retry_count_429:,
+ retry_count_transient:
+ )
+ status = response.code.to_i
+ retry_delay =
+ retry_delay_for_response(
+ response,
+ retry_count_429: retry_count_429,
+ retry_count_transient: retry_count_transient,
+ )
+
+ log_method = retry_delay ? :warn : :error
+ Rails.logger.public_send(
+ log_method,
+ "#{self.class.name}: status: #{status} - body: #{response.body}",
+ )
+ response_raw << response.body.to_s
+
+ [status, retry_delay]
+ end
+
+ def build_xml_tool_processor(dialect, partial_tool_calls)
+ return if !xml_tools_enabled? || !dialect.prompt.has_tools?
+
+ XmlToolProcessor.new(
+ partial_tool_calls: partial_tool_calls,
+ tool_definitions: dialect.prompt.tools,
+ )
+ end
+
+ def build_xml_stripper(dialect)
+ to_strip = xml_tags_to_strip(dialect)
+ return if to_strip.blank?
+
+ DiscourseAi::Completions::XmlTagStripper.new(to_strip)
+ end
+
+ def streaming_partial_handler(orig_blk, xml_stripper, structured_output)
+ lambda do |partial|
+ if partial.is_a?(String)
+ partial = xml_stripper << partial if xml_stripper && !partial.empty?
+
+ if structured_output.present?
+ structured_output << partial if !partial.empty?
+ partial = structured_output
+ end
+ end
+ orig_blk.call(partial) if partial
+ end
+ end
+
+ def streaming_response(
+ response:,
+ blk:,
+ xml_tool_processor:,
+ xml_stripper:,
+ partials_raw:,
+ response_raw:,
+ structured_output:,
+ cancelled:,
+ on_output_started:
+ )
+ response_data = +""
+
+ response.read_body do |chunk|
+ break if cancelled.call
+
+ response_raw << chunk
+
+ decode_chunk(chunk).each do |partial|
+ break if cancelled.call
+ partials_raw << partial.to_s
+ response_data << partial if partial.is_a?(String)
+ partials = [partial]
+ if xml_tool_processor && partial.is_a?(String)
+ partials = (xml_tool_processor << partial)
+ break if xml_tool_processor.should_cancel?
+ end
+ partials.each do |inner_partial|
+ on_output_started.call
+ blk.call(inner_partial)
+ end
+ end
+ end
+
+ finish_streaming_response(
+ response_data: response_data,
+ blk: blk,
+ xml_tool_processor: xml_tool_processor,
+ xml_stripper: xml_stripper,
+ structured_output: structured_output,
+ )
+
+ response_data
+ end
+
+ def finish_streaming_response(
+ response_data:,
+ blk:,
+ xml_tool_processor:,
+ xml_stripper:,
+ structured_output:
+ )
+ if xml_stripper
+ stripped = xml_stripper.finish
+ if stripped.present?
+ response_data << stripped
+ result = []
+ result = (xml_tool_processor << stripped) if xml_tool_processor
+ result.each { |partial| blk.call(partial) }
+ end
+ end
+
+ xml_tool_processor.finish.each { |partial| blk.call(partial) } if xml_tool_processor
+ decode_chunk_finish.each { |partial| blk.call(partial) }
+
+ finish_structured_output_streaming(structured_output, blk) if structured_output
+ end
+
+ def finish_structured_output_streaming(structured_output, blk)
+ structured_output.finish
+ if structured_output.broken?
+ # signal last partial output which will get parsed
+ # by best effort json parser
+ blk.call("")
+ else
+ # got to signal the end of structured output
+ blk.call(structured_output)
+ end
+ end
+
+ def persist_completion_log!(
+ log,
+ response_raw:,
+ partials_raw:,
+ request_attempts:,
+ call_status:,
+ start_time:,
+ feature_name:,
+ user:,
+ execution_context:
+ )
+ log.raw_response_payload = response_raw
+ log.request_attempts = request_attempts if log.has_attribute?(:request_attempts)
+ final_log_update(log)
+ log.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?
+ log.response_status ||= 200 if call_status == :success
+ log.created_at = start_time
+ log.updated_at = Time.now
+ log.duration_msecs = (Time.now - start_time) * 1000
+ log.save!
+
+ execution_context&.token_usage_tracker&.add_from_audit_log(log)
+
+ AiApiRequestStat.record_from_audit_log(log, llm_model: @llm_model)
+ LlmQuota.log_usage(@llm_model, user, log.request_tokens, log.response_tokens)
+ LlmCreditAllocation.deduct_credits!(
+ @llm_model,
+ feature_name,
+ log.request_tokens,
+ log.response_tokens,
+ )
+
+ DiscourseAi::Completions::LlmMetric.record(
+ llm_model: @llm_model,
+ feature_name: feature_name,
+ request_tokens: log.request_tokens || 0,
+ response_tokens: log.response_tokens || 0,
+ duration_ms: log.duration_msecs,
+ status: call_status,
+ )
+
+ if Rails.env.development? && ENV["DISCOURSE_AI_DEBUG"]
+ puts "#{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"
+ end
+ end
+
+ def log_completion_audit_entries(
+ log,
+ request:,
+ response_data:,
+ start_time:,
+ execution_context:
+ )
+ log_text_audit_entry(log, response_data, execution_context&.audit_logger)
+ log_structured_audit_entry(
+ log,
+ request: request,
+ response_data: response_data,
+ start_time: start_time,
+ structured_logger: execution_context&.structured_audit_logger,
+ )
+ end
+
+ def log_text_audit_entry(log, response_data, logger)
+ return if !logger
+
+ call_data = <<~LOG
+ #{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}
+ request:
+ #{format_possible_json_payload(log.raw_request_payload)}
+ response:
+ #{response_data}
+ LOG
+ logger.info(call_data)
+ end
+
+ def log_structured_audit_entry(
+ log,
+ request:,
+ response_data:,
+ start_time:,
+ structured_logger:
+ )
+ return if !structured_logger
+
+ llm_request =
+ begin
+ JSON.parse(log.raw_request_payload)
+ rescue StandardError
+ log.raw_request_payload
+ end
+
+ # gemini puts passwords in query params
+ # we don't want to log that
+ llm_call_step = structured_logger.add_child_step(name: "Performing LLM call")
+
+ structured_logger.append_entry(
+ step: llm_call_step,
+ name: "llm_call",
+ args: {
+ class: self.class.name,
+ completion_url: request.uri.to_s.split("?")[0],
+ request: llm_request,
+ result: response_data,
+ request_tokens: log.request_tokens,
+ response_tokens: log.response_tokens,
+ duration: log.duration_msecs,
+ stream: @streaming_mode,
+ },
+ started_at: start_time.utc,
+ ended_at: Time.now.utc,
+ )
+ end
+
+ def retry_delay_for_response(response, retry_count_429:, retry_count_transient:)
+ status = response.code.to_i
+
+ if status == 429 && retry_count_429 < RATE_LIMIT_RETRY_DELAYS.length
+ delay = RATE_LIMIT_RETRY_DELAYS[retry_count_429]
+ retry_after_delay = [
+ retry_after_delay(response["Retry-After"]),
+ retry_delay_from_response_body(response.body),
+ ].compact.max
+
+ return retry_delay_with_jitter([delay, retry_after_delay].compact.max)
+ end
+
+ if transient_error_status?(status) &&
+ retry_count_transient < TRANSIENT_ERROR_RETRY_DELAYS.length
+ delay = TRANSIENT_ERROR_RETRY_DELAYS[retry_count_transient]
+ retry_after_delay = [
+ retry_after_delay(response["Retry-After"]),
+ retry_delay_from_response_body(response.body),
+ ].compact.max
+
+ return retry_delay_with_jitter([delay, retry_after_delay].compact.max)
+ end
+
+ nil
+ end
+
+ def retry_delay_for_network_error(retry_count_transient:)
+ return if retry_count_transient >= TRANSIENT_ERROR_RETRY_DELAYS.length
+
+ TRANSIENT_ERROR_RETRY_DELAYS[retry_count_transient] + retry_jitter
+ end
+
+ def retry_delay_with_jitter(delay)
+ [delay + retry_jitter, MAX_RETRY_AFTER_SECONDS].min
+ end
+
+ def transient_error_status?(status)
+ status == 408 || status == 409 || (status >= 500 && status < 600)
+ end
+
+ def sleep_before_retry(delay, cancel_manager = nil)
+ if cancel_manager
+ cancel_manager.wait_for_cancel(delay) if !cancel_manager.cancelled?
+ else
+ sleep(delay)
+ end
+ end
+
+ def retry_after_delay(header)
+ return if header.blank?
+
+ delay =
+ if header.to_s.match?(/\A\d+(\.\d+)?\z/)
+ header.to_f
+ else
+ Time.httpdate(header).to_f - Time.now.to_f
+ end
+
+ return if delay <= 0
+
+ [delay, MAX_RETRY_AFTER_SECONDS].min
+ rescue StandardError
+ nil
+ end
+
+ def retry_delay_from_response_body(_body)
+ nil
+ end
+
+ def retry_jitter
+ rand * RETRY_JITTER_MAX_SECONDS
+ end
+
def format_possible_json_payload(payload)
JSON.pretty_generate(JSON.parse(payload))
rescue JSON::ParserError
diff --git a/plugins/discourse-ai/lib/completions/endpoints/gemini.rb b/plugins/discourse-ai/lib/completions/endpoints/gemini.rb
index 1c0275a0013..d13ef5bf0fd 100644
--- a/plugins/discourse-ai/lib/completions/endpoints/gemini.rb
+++ b/plugins/discourse-ai/lib/completions/endpoints/gemini.rb
@@ -139,15 +139,45 @@ module DiscourseAi
payload[:generationConfig][:thinkingConfig][:includeThoughts] = true
end
+ payload[:serviceTier] = service_tier if service_tier.present?
+
payload
end
+ def service_tier
+ return @service_tier if defined?(@service_tier)
+
+ @service_tier = llm_model.lookup_custom_param("service_tier")
+ @service_tier = nil if !%w[standard flex priority].include?(@service_tier)
+ @service_tier
+ end
+
def prepare_request(payload)
headers = { "Content-Type" => "application/json" }
Net::HTTP::Post.new(model_uri, headers).tap { |r| r.body = payload }
end
+ def retry_delay_from_response_body(body)
+ return if body.blank? || body.bytesize >= 10_000
+
+ retry_info =
+ Array(JSON.parse(body).dig("error", "details")).find do |detail|
+ detail["@type"] == "type.googleapis.com/google.rpc.RetryInfo"
+ end
+
+ retry_delay = retry_info&.[]("retryDelay")
+ match = retry_delay.to_s.match(/\A(?<seconds>\d+(?:\.\d+)?)s\z/)
+ return if !match
+
+ delay = match[:seconds].to_f
+ return if delay <= 0
+
+ [delay, MAX_RETRY_AFTER_SECONDS].min
+ rescue StandardError
+ nil
+ end
+
def extract_completion_from(response_raw)
parsed =
if @streaming_mode
diff --git a/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rb b/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rb
index 5d1d8661ced..784ec8d6065 100644
--- a/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rb
@@ -36,6 +36,30 @@ describe DiscourseAi::Completions::CancelManager do
expect(cancel_manager.monitor_thread).to be_nil
end
+ it "waits until timeout when not cancelled" do
+ cancel_manager = DiscourseAi::Completions::CancelManager.new
+
+ expect(cancel_manager.wait_for_cancel(0.001)).to eq(false)
+ end
+
+ it "wakes waiters when cancelled" do
+ cancel_manager = DiscourseAi::Completions::CancelManager.new
+ waiting = Queue.new
+
+ waiter =
+ Thread.new do
+ waiting << true
+ cancel_manager.wait_for_cancel(60)
+ end
+
+ waiting.pop
+ cancel_manager.cancel!
+
+ expect(waiter.join(1)).to eq(waiter)
+ ensure
+ waiter&.kill
+ end
+
it "should do nothing when cancel manager is already cancelled" do
cancel_manager = DiscourseAi::Completions::CancelManager.new
cancel_manager.cancel!
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
index 29e17da8f91..8950de19072 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb
@@ -312,6 +312,23 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse do
expect { llm.generate("hello", user: user) }.to raise_error(
DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
)
+ expect(AiApiAuditLog.last.response_status).to be_nil
+ end
+ it "records SDK error response status when available" do
+ client = instance_double(Aws::BedrockRuntime::Client)
+ context = Seahorse::Client::RequestContext.new
+ context.http_response.status_code = 429
+ allow(client).to receive(:converse).and_raise(
+ Aws::BedrockRuntime::Errors::ThrottlingException.new(context, "Rate exceeded"),
+ )
+ allow(Aws::BedrockRuntime::Client).to receive(:new).and_return(client)
+
+ llm = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")
+
+ expect { llm.generate("hello", user: user) }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ expect(AiApiAuditLog.last.response_status).to eq(429)
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb
index 239a2101249..b3441a5c123 100644
--- a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb
@@ -188,8 +188,210 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Gemini do
}
end
+ def gemini_rate_limit_body(retry_delay: "45s")
+ {
+ error: {
+ code: 429,
+ message: "quota exceeded",
+ status: "RESOURCE_EXHAUSTED",
+ details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: retry_delay }],
+ },
+ }.to_json
+ end
+
before { enable_current_plugin }
+ it "uses Gemini RetryInfo retryDelay from rate limit response bodies" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(45, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ request =
+ stub_request(:post, url).to_return(
+ { status: 429, body: gemini_rate_limit_body(retry_delay: "45s") },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ expect(request).to have_been_requested.times(2)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 45_000 }],
+ )
+ end
+
+ it "caps Gemini RetryInfo retryDelay from rate limit response bodies" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(60, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ stub_request(:post, url).to_return(
+ { status: 429, body: gemini_rate_limit_body(retry_delay: "120s") },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
+ it "ignores oversized Gemini rate limit response bodies when extracting retry delays" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(2, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ stub_request(:post, url).to_return(
+ { status: 429, body: gemini_rate_limit_body(retry_delay: "45s") + (" " * 10_000) },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
+ it "uses the larger retry delay from Gemini RetryInfo and Retry-After" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(10, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ stub_request(:post, url).to_return(
+ {
+ status: 429,
+ body: gemini_rate_limit_body(retry_delay: "10s"),
+ headers: {
+ "Retry-After" => "5",
+ },
+ },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
+ it "uses Gemini RetryInfo retryDelay for transient errors" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(12, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ stub_request(:post, url).to_return(
+ { status: 503, body: gemini_rate_limit_body(retry_delay: "12s") },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
+ it "supports fractional Gemini RetryInfo retryDelay values" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(2.5, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ stub_request(:post, url).to_return(
+ { status: 429, body: gemini_rate_limit_body(retry_delay: "2.5s") },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
+ it "ignores non-positive Gemini RetryInfo retryDelay values" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(2, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ stub_request(:post, url).to_return(
+ { status: 429, body: gemini_rate_limit_body(retry_delay: "0s") },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
+ it "finds Gemini RetryInfo among other error details" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(7, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ body = {
+ error: {
+ code: 429,
+ details: [
+ { "@type": "type.googleapis.com/google.rpc.Help" },
+ { "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "7s" },
+ ],
+ },
+ }.to_json
+ stub_request(:post, url).to_return(
+ { status: 429, body: body },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
+ it "ignores invalid Gemini rate limit response bodies when extracting retry delays" do
+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Gemini
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(2, nil)
+ .once
+
+ url = "#{model.url}:generateContent?key=123"
+ stub_request(:post, url).to_return(
+ { status: 429, body: "{" },
+ { status: 200, body: gemini_mock.response("ok").to_json },
+ )
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+
+ expect(llm.generate("Hello", user: user)).to eq("ok")
+ end
+
it "correctly configures thinking when enabled" do
model.update!(provider_params: { enable_thinking: "true", thinking_tokens: "10000" })
@@ -476,6 +678,81 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Gemini do
expect(parsed.dig(:generationConfig, :thinkingConfig)).to be_nil
end
+ it "adds configured service tier using the Gemini API field name" do
+ response = gemini_mock.response("Configured service tier").to_json
+ url = "#{model.url}:generateContent?key=123"
+
+ captured_bodies = []
+ stub_request(:post, url).with(
+ body:
+ proc do |req_body|
+ captured_bodies << JSON.parse(req_body, symbolize_names: true)
+ true
+ end,
+ ).to_return(status: 200, body: response)
+
+ {
+ "default" => nil,
+ "standard" => "standard",
+ "flex" => "flex",
+ "priority" => "priority",
+ }.each do |configured_tier, expected_tier|
+ model.update!(provider_params: { service_tier: configured_tier })
+
+ DiscourseAi::Completions::Llm.proxy(model).generate("Hello", user: user)
+
+ payload = captured_bodies.last
+ if expected_tier
+ expect(payload[:serviceTier]).to eq(expected_tier)
+ else
+ expect(payload).not_to have_key(:serviceTier)
+ end
+ expect(payload).not_to have_key(:service_tier)
+ end
+ end
+
+ it "omits service tier when it is not configured" do
+ response = gemini_mock.response("No service tier").to_json
+ req_body = nil
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+ url = "#{model.url}:generateContent?key=123"
+
+ stub_request(:post, url).with(
+ body:
+ proc do |_req_body|
+ req_body = _req_body
+ true
+ end,
+ ).to_return(status: 200, body: response)
+
+ llm.generate("Hello", user: user)
+
+ expect(JSON.parse(req_body, symbolize_names: true)).not_to have_key(:serviceTier)
+ end
+
+ it "omits service tier when it is invalid" do
+ model.update!(provider_params: { service_tier: "invalid" })
+
+ response = gemini_mock.response("Invalid service tier").to_json
+ req_body = nil
+
+ llm = DiscourseAi::Completions::Llm.proxy(model)
+ url = "#{model.url}:generateContent?key=123"
+
+ stub_request(:post, url).with(
+ body:
+ proc do |_req_body|
+ req_body = _req_body
+ true
+ end,
+ ).to_return(status: 200, body: response)
+
+ llm.generate("Hello", user: user)
+
+ expect(JSON.parse(req_body, symbolize_names: true)).not_to have_key(:serviceTier)
+ end
+
it "strips temperature when thinking_level is set" do
model.update!(provider_params: { enable_thinking: true, thinking_level: "high" })
diff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
index ea6865aaf66..e25aadbdd2b 100644
--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb
@@ -29,7 +29,7 @@ RSpec.describe DiscourseAi::Completions::Llm do
def streaming_body(content: "Hello")
<<~SSE
- data: {"id":"1","object":"chat.completion.chunk","choices":[{"delta":{"content":"#{content}"}}]}
+ data: {"id":"1","object":"chat.completion.chunk","choices":[{"delta":{"content":#{content.to_json}}}]}
data: [DONE]
SSE
@@ -63,6 +63,17 @@ RSpec.describe DiscourseAi::Completions::Llm do
llm.generate("hi", user:) { |partial| result << partial }
expect(result).to eq("Hi")
end
+
+ it "replays non-streaming responses when streaming is disabled" do
+ model.update!(provider_params: { "disable_streaming" => true })
+ stub_response(body: success_body(content: "Hi"))
+
+ partials = []
+ result = llm.generate("hi", user:) { |partial| partials << partial }
+
+ expect(result).to eq("Hi")
+ expect(partials).to eq(["Hi"])
+ end
end
context "with a fake model" do
@@ -87,6 +98,33 @@ RSpec.describe DiscourseAi::Completions::Llm do
end
end
+ context "with structured output" do
+ it "returns a structured output buffer" do
+ stub_response(body: success_body(content: '{"message":"ok"}'))
+
+ result =
+ llm.generate(
+ "hello",
+ user:,
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ )
+
+ expect(result).to be_a(DiscourseAi::Completions::StructuredOutput)
+ expect(result).to be_finished
+ expect(result.to_s).to eq('{"message":"ok"}')
+ end
+ end
+
context "when auditing" do
it "logs topic_id, post_id, feature_name, and feature_context" do
stub_response(body: success_body)
@@ -169,9 +207,436 @@ RSpec.describe DiscourseAi::Completions::Llm do
end
end
+ context "when retrying failed requests" do
+ before do
+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:sleep_before_retry)
+ end
+
+ it "retries rate limits three times" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 429, body: "rate limited" },
+ { status: 429, body: "rate limited" },
+ { status: 429, body: "rate limited" },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ result = nil
+
+ expect { result = llm.generate("Hello", user:) }.to change { AiApiAuditLog.count }.by(1)
+
+ expect(result).to eq("ok")
+ expect(request).to have_been_requested.times(4)
+ expect(AiApiAuditLog.last.response_status).to eq(200)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [
+ { "status" => 429, "delay_ms" => 0 },
+ { "status" => 429, "delay_ms" => 2000 },
+ { "status" => 429, "delay_ms" => 8000 },
+ { "status" => 200, "delay_ms" => 16_000 },
+ ],
+ )
+ end
+
+ it "does not retry non-retryable client errors" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(status: 401, body: "unauthorized")
+
+ expect { llm.generate("Hello", user:) }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ expect(request).to have_been_requested.once
+ expect(AiApiAuditLog.last.response_status).to eq(401)
+ expect(AiApiAuditLog.last.request_attempts).to be_nil
+ end
+
+ it "includes retry waits in audit duration" do
+ start_time = Time.utc(2026, 1, 1, 12, 0, 0)
+ current_time = start_time
+ Time.stubs(:now).returns(current_time)
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with do |delay, cancel_manager|
+ current_time += delay.seconds
+ Time.stubs(:now).returns(current_time)
+ delay == 2 && cancel_manager.nil?
+ end
+ .once
+
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 429, body: "rate limited" },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],
+ )
+ expect(AiApiAuditLog.last.duration_msecs).to be >= 2000
+ end
+
+ it "respects retry-after for rate limits" do
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(5, nil)
+ .once
+
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 429, body: "rate limited", headers: { "Retry-After" => "5" } },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ end
+
+ it "respects retry-after HTTP dates for rate limits" do
+ freeze_time
+
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with { |delay, cancel_manager| delay.between?(29, 30) && cancel_manager.nil? }
+ .once
+
+ WebMock.stub_request(:post, model.url).to_return(
+ {
+ status: 429,
+ body: "rate limited",
+ headers: {
+ "Retry-After" => 30.seconds.from_now.httpdate,
+ },
+ },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ end
+
+ it "caps retry-after values after adding jitter" do
+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:retry_jitter).returns(1)
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(60, nil)
+ .once
+
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 429, body: "rate limited", headers: { "Retry-After" => "999999" } },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ end
+
+ it "raises rate limits after three retries" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(status: 429, body: "rate limited")
+
+ expect { llm.generate("Hello", user:) }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ expect(request).to have_been_requested.times(4)
+ expect(AiApiAuditLog.last.response_status).to eq(429)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [
+ { "status" => 429, "delay_ms" => 0 },
+ { "status" => 429, "delay_ms" => 2000 },
+ { "status" => 429, "delay_ms" => 8000 },
+ { "status" => 429, "delay_ms" => 16_000 },
+ ],
+ )
+ end
+
+ it "retries streaming responses after rate limits" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 429, body: "rate limited" },
+ { status: 200, body: streaming_body(content: "Hi") },
+ )
+
+ result = +""
+ llm.generate("Hello", user:) { |partial| result << partial }
+
+ expect(result).to eq("Hi")
+ expect(request).to have_been_requested.times(2)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],
+ )
+ end
+
+ it "does not retry streaming responses after output has started" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(status: 200, body: streaming_body)
+
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:streaming_response)
+ .with do |kwargs|
+ kwargs[:on_output_started].call
+ kwargs[:blk].call("partial")
+ true
+ end
+ .raises(Net::ReadTimeout.new("timed out"))
+
+ result = +""
+ expect { llm.generate("Hello", user:) { |partial| result << partial } }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ expect(result).to eq("partial")
+ expect(request).to have_been_requested.once
+ expect(AiApiAuditLog.last.response_status).to eq(200)
+ expect(AiApiAuditLog.last.request_attempts).to be_nil
+ end
+
+ it "retries streaming structured output after rate limits" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 429, body: "rate limited" },
+ { status: 200, body: streaming_body(content: '{"message":"ok"}') },
+ )
+
+ result = nil
+ llm.generate(
+ "Hello",
+ user:,
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ ) { |partial| result = partial }
+
+ expect(result).to be_a(DiscourseAi::Completions::StructuredOutput)
+ expect(result.to_s).to eq('{"message":"ok"}')
+ expect(request).to have_been_requested.times(2)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],
+ )
+ end
+
+ it "returns structured output after rate limits" do
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 429, body: "rate limited" },
+ { status: 200, body: success_body(content: '{"message":"ok"}').to_json },
+ )
+
+ result =
+ llm.generate(
+ "Hello",
+ user:,
+ response_format: {
+ json_schema: {
+ schema: {
+ properties: {
+ message: {
+ type: "string",
+ },
+ },
+ },
+ },
+ },
+ )
+
+ expect(result).to be_a(DiscourseAi::Completions::StructuredOutput)
+ expect(result.to_s).to eq('{"message":"ok"}')
+ end
+
+ it "retries network errors twice" do
+ request =
+ WebMock
+ .stub_request(:post, model.url)
+ .to_raise(Net::ReadTimeout.new("timed out"))
+ .then
+ .to_raise(Errno::ECONNRESET.new)
+ .then
+ .to_return(status: 200, body: success_body(content: "ok").to_json)
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ expect(request).to have_been_requested.times(3)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [
+ { "status" => 0, "delay_ms" => 0 },
+ { "status" => 0, "delay_ms" => 500 },
+ { "status" => 200, "delay_ms" => 1000 },
+ ],
+ )
+ end
+
+ it "raises network errors after two retries" do
+ request = WebMock.stub_request(:post, model.url).to_raise(Net::ReadTimeout.new("timed out"))
+
+ expect { llm.generate("Hello", user:) }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ expect(request).to have_been_requested.times(3)
+ expect(AiApiAuditLog.last.response_status).to be_nil
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [
+ { "status" => 0, "delay_ms" => 0 },
+ { "status" => 0, "delay_ms" => 500 },
+ { "status" => 0, "delay_ms" => 1000 },
+ ],
+ )
+ end
+
+ it "retries request timeouts twice" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 408, body: "timeout" },
+ { status: 408, body: "timeout" },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ expect(request).to have_been_requested.times(3)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [
+ { "status" => 408, "delay_ms" => 0 },
+ { "status" => 408, "delay_ms" => 500 },
+ { "status" => 200, "delay_ms" => 1000 },
+ ],
+ )
+ end
+
+ it "retries lock timeouts twice" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 409, body: "conflict" },
+ { status: 409, body: "conflict" },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ expect(request).to have_been_requested.times(3)
+ end
+
+ it "retries server errors twice" do
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(0.5, nil)
+ .once
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(1.0, nil)
+ .once
+
+ request =
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 503, body: "unavailable" },
+ { status: 503, body: "unavailable" },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ expect(request).to have_been_requested.times(3)
+ end
+
+ it "respects retry-after for server errors" do
+ DiscourseAi::Completions::Endpoints::Base
+ .any_instance
+ .expects(:sleep_before_retry)
+ .with(5, nil)
+ .once
+
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 503, body: "unavailable", headers: { "Retry-After" => "5" } },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [{ "status" => 503, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 5000 }],
+ )
+ end
+
+ it "tracks mixed request attempts" do
+ request =
+ WebMock.stub_request(:post, model.url).to_return(
+ { status: 503, body: "unavailable" },
+ { status: 429, body: "rate limited" },
+ { status: 200, body: success_body(content: "ok").to_json },
+ )
+
+ expect(llm.generate("Hello", user:)).to eq("ok")
+ expect(request).to have_been_requested.times(3)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [
+ { "status" => 503, "delay_ms" => 0 },
+ { "status" => 429, "delay_ms" => 500 },
+ { "status" => 200, "delay_ms" => 2000 },
+ ],
+ )
+ end
+
+ it "raises server errors after two retries" do
+ request = WebMock.stub_request(:post, model.url).to_return(status: 503, body: "unavailable")
+
+ expect { llm.generate("Hello", user:) }.to raise_error(
+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,
+ )
+ expect(request).to have_been_requested.times(3)
+ expect(AiApiAuditLog.last.response_status).to eq(503)
+ expect(AiApiAuditLog.last.request_attempts).to eq(
+ [
+ { "status" => 503, "delay_ms" => 0 },
+ { "status" => 503, "delay_ms" => 500 },
+ { "status" => 503, "delay_ms" => 1000 },
+ ],
+ )
+ end
+ end
+
+ context "when sleeping before retries" do
+ it "sleeps normally without a cancel manager" do
+ endpoint = DiscourseAi::Completions::Endpoints::Base.new(model)
+ endpoint.expects(:sleep).with(3).once
+
+ endpoint.send(:sleep_before_retry, 3, nil)
+ end
+
+ it "stops when cancelled" do
+ cancel_manager = DiscourseAi::Completions::CancelManager.new
+ endpoint = DiscourseAi::Completions::Endpoints::Base.new(model)
+ waiting = Queue.new
+
+ sleep_thread =
+ Thread.new do
+ waiting << true
+ endpoint.send(:sleep_before_retry, 60, cancel_manager)
+ end
+
+ waiting.pop
+ cancel_manager.cancel!
+
+ expect(sleep_thread.join(1)).to eq(sleep_thread)
+ ensure
+ sleep_thread&.kill
+ end
+ end
+
context "when tracking failures" do
+ before do
+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:retry_jitter).returns(0)
+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:sleep_before_retry)
+ end
+
it "fast-tracks problem check after threshold and resets on success" do
WebMock.stub_request(:post, model.url).to_return(
+ { status: 500, body: "fail" },
+ { status: 500, body: "fail" },
+ { status: 500, body: "fail" },
+ { status: 500, body: "fail" },
{ status: 500, body: "fail" },
{ status: 500, body: "fail" },
{ status: 200, body: success_body.to_json },
diff --git a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb
index 21ca7f1c5fe..5146ea3f84e 100644
--- a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb
+++ b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb
@@ -84,6 +84,18 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController do
expect(vllm_params).not_to have_key("enable_thinking")
end
+ it "includes Gemini service tier metadata" do
+ get "/admin/plugins/discourse-ai/ai-llms.json"
+ expect(response).to be_successful
+
+ google_params = response.parsed_body.dig("meta", "provider_params", "google")
+ expect(google_params["service_tier"]).to include(
+ "type" => "enum",
+ "values" => %w[default standard flex priority],
+ "default" => "default",
+ )
+ end
+
it "lists enabled features on appropriate LLMs" do
SiteSetting.ai_bot_enabled = true
SiteSetting.ai_bot_enabled_llms = llm_model.id.to_s
diff --git a/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb b/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb
new file mode 100644
index 00000000000..48e13aa3f25
--- /dev/null
+++ b/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+RSpec.describe AiApiAuditLogSerializer do
+ before { enable_current_plugin }
+
+ it "omits request attempts when the column is unavailable" do
+ log = AiApiAuditLog.new(provider_id: AiApiAuditLog::Provider::OpenAI)
+ original_has_attribute = log.method(:has_attribute?)
+ log.define_singleton_method(:has_attribute?) do |attribute|
+ attribute.to_sym == :request_attempts ? false : original_has_attribute.call(attribute)
+ end
+
+ serialized = described_class.new(log).as_json[:ai_api_audit_log]
+
+ expect(serialized).not_to have_key(:request_attempts)
+ end
+
+ it "includes request attempts when the column is available" do
+ log =
+ AiApiAuditLog.new(
+ provider_id: AiApiAuditLog::Provider::OpenAI,
+ response_status: 200,
+ request_attempts: [
+ { "status" => 429, "delay_ms" => 0 },
+ { "status" => 200, "delay_ms" => 2000 },
+ ],
+ )
+
+ serialized = described_class.new(log).as_json[:ai_api_audit_log]
+
+ expect(serialized[:request_attempts]).to eq(
+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],
+ )
+ expect(serialized[:response_status]).to eq(200)
+ end
+end