A platform for community discussion. Free, open, simple.
FEATURE: retry transient and rate-limited LLM completion requests (#41237)
Sam3d737f966e25
diff --git a/db/structure.sql b/db/structure.sqlindex 9ca2acbd86b..88e9935df6a 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -517,7 +517,8 @@ CREATE TABLE public.ai_api_audit_logs (cache_write_tokens integer,cache_read_tokens integer,llm_id bigint,- response_status integer+ response_status integer,+ request_attempts jsonb);@@ -22274,6 +22275,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260629022603'),('20260626055145'),('20260624140945'),('20260623090824'),diff --git a/plugins/discourse-ai/app/models/ai_api_audit_log.rb b/plugins/discourse-ai/app/models/ai_api_audit_log.rbindex 612c66b661e..175df4f4315 100644--- a/plugins/discourse-ai/app/models/ai_api_audit_log.rb+++ b/plugins/discourse-ai/app/models/ai_api_audit_log.rb@@ -1,7 +1,9 @@# frozen_string_literal: trueclass AiApiAuditLog < ActiveRecord::Base- self.ignored_columns = %w[cached_tokens] # TODO: Remove when 20251118000500_drop_cached_tokens_from_ai_api_audit_logs has been promoted to pre-deploy+ self.ignored_columns += [+ "cached_tokens", # TODO: Remove when 20251118000500_drop_cached_tokens_from_ai_api_audit_logs has been promoted to pre-deploy+ ]belongs_to :postbelongs_to :topicbelongs_to :user@@ -64,6 +66,7 @@ end# language_model :string(255)# raw_request_payload :string# raw_response_payload :string+# request_attempts :jsonb# request_tokens :integer# response_status :integer# response_tokens :integerdiff --git a/plugins/discourse-ai/app/models/llm_model.rb b/plugins/discourse-ai/app/models/llm_model.rbindex 65e1cf159cf..34b1dc5e2e1 100644--- a/plugins/discourse-ai/app/models/llm_model.rb+++ b/plugins/discourse-ai/app/models/llm_model.rb@@ -258,6 +258,11 @@ class LlmModel < ActiveRecord::Basehidden_if: :enable_thinking,},disable_top_p: :checkbox,+ service_tier: {+ type: :enum,+ values: %w[default standard flex priority],+ default: "default",+ },},azure: {disable_native_tools: :checkbox,diff --git a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rbindex a4a1c74af8c..33d705eeae5 100644--- a/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb+++ b/plugins/discourse-ai/app/serializers/ai_api_audit_log_serializer.rb@@ -15,6 +15,8 @@ class AiApiAuditLogSerializer < ApplicationSerializer:feature_name,:llm_id,:language_model,+ :response_status,+ :request_attempts,:created_at,:prev_log_id,:next_log_id,@@ -25,6 +27,14 @@ class AiApiAuditLogSerializer < ApplicationSerializer:conversation_cache_write_tokens,:conversation_spending+ def request_attempts+ object.request_attempts+ end++ def include_request_attempts?+ object.has_attribute?(:request_attempts)+ end+def spendingobject.llm_model&.spending_for(object)enddiff --git a/plugins/discourse-ai/db/migrate/20260629022603_add_request_attempts_to_ai_api_audit_logs.rb b/plugins/discourse-ai/db/migrate/20260629022603_add_request_attempts_to_ai_api_audit_logs.rbnew file mode 100644index 00000000000..3e30398ac09--- /dev/null+++ b/plugins/discourse-ai/db/migrate/20260629022603_add_request_attempts_to_ai_api_audit_logs.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++class AddRequestAttemptsToAiApiAuditLogs < ActiveRecord::Migration[8.0]+ def up+ unless column_exists?(:ai_api_audit_logs, :request_attempts)+ add_column :ai_api_audit_logs, :request_attempts, :jsonb+ end+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/plugins/discourse-ai/lib/completions/cancel_manager.rb b/plugins/discourse-ai/lib/completions/cancel_manager.rbindex 82ce562b21c..e3e5322db08 100644--- a/plugins/discourse-ai/lib/completions/cancel_manager.rb+++ b/plugins/discourse-ai/lib/completions/cancel_manager.rb@@ -9,6 +9,7 @@ module DiscourseAidef initialize@cancelled = false+ @cancel_event = Concurrent::Event.new@callbacks = Concurrent::Array.new@mutex = Mutex.new@monitor_thread = nil@@ -79,12 +80,17 @@ module DiscourseAi@callbacks << cbend+ def wait_for_cancel(timeout)+ @cancel_event.wait(timeout)+ end+def remove_callback(cb)@callbacks.delete(cb)enddef cancel!@cancelled = true+ @cancel_event.setmonitor_thread = @monitor_threadif monitor_thread && monitor_thread != Thread.currentmonitor_thread.wakeupdiff --git a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rbindex 5e387c824a2..847dcd8c670 100644--- a/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb+++ b/plugins/discourse-ai/lib/completions/endpoints/aws_bedrock_converse.rb@@ -176,6 +176,7 @@ module DiscourseAiresponse_dataendrescue Aws::BedrockRuntime::Errors::ServiceError => e+ log.response_status = e.context&.http_response&.status_code if logRails.logger.error("#{self.class.name}: #{e.class}: #{e.message}")raise CompletionFailed, e.messageensure@@ -190,7 +191,7 @@ module DiscourseAilog.cache_write_tokens =processor.cache_write_input_tokens if processor.cache_write_input_tokenslog.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?- log.response_status ||= 200+ log.response_status ||= 200 if call_status == :successlog.created_at = start_timelog.updated_at = Time.nowlog.duration_msecs = (Time.now - start_time) * 1000@@ -277,6 +278,9 @@ module DiscourseAiregion = llm_model.lookup_custom_param("region")client_options = { region: region, http_read_timeout: TIMEOUT }+ # AWS SDK already retries modeled throttling, 5xx, and networking failures+ # by default. Keep Bedrock Converse on SDK-managed retries to avoid+ # double-retrying request streams in Discourse's HTTP retry loop.role_arn = llm_model.lookup_custom_param("role_arn")access_key_id = llm_model.lookup_custom_param("access_key_id")diff --git a/plugins/discourse-ai/lib/completions/endpoints/base.rb b/plugins/discourse-ai/lib/completions/endpoints/base.rbindex fd5850fbbd6..d9aa0b6dcb7 100644--- a/plugins/discourse-ai/lib/completions/endpoints/base.rb+++ b/plugins/discourse-ai/lib/completions/endpoints/base.rb@@ -9,6 +9,24 @@ module DiscourseAiCompletionFailed = Class.new(StandardError)FAIL_THRESHOLD = 5FAIL_TTL = 1.hour+ # Applies to both synchronous and background completions; keep retry waits bounded+ # and interruptible via sleep_before_retry.+ RATE_LIMIT_RETRY_DELAYS = [2, 8, 16]+ RETRY_JITTER_MAX_SECONDS = 1.0+ TRANSIENT_ERROR_RETRY_DELAYS = [0.5, 1.0]+ MAX_RETRY_AFTER_SECONDS = 60+ # Stored in AiApiAuditLog#request_attempts for retried network failures,+ # which do not have an HTTP status code. 0 is intentionally outside the HTTP range.+ NETWORK_ERROR_RETRY_STATUS = 0+ RETRIABLE_NETWORK_ERRORS = [+ Net::OpenTimeout,+ Net::ReadTimeout,+ Errno::ECONNREFUSED,+ Errno::ECONNRESET,+ Errno::ETIMEDOUT,+ SocketError,+ OpenSSL::SSL::SSLError,+ ]# Read-only context handed to request header providers. Lets plugins# contribute provider-agnostic, per-request HTTP headers (e.g. routing@@ -118,8 +136,6 @@ module DiscourseAiLlmQuota.check_quotas!(@llm_model, user)LlmCreditAllocation.check_credits!(@llm_model, feature_name)- start_time = Time.now-if cancel_manager && cancel_manager.cancelled?# nothing to doreturn@@ -134,11 +150,10 @@ module DiscourseAimax_tokens = enforce_max_output_tokens(model_params[:max_tokens])model_params[:max_tokens] = max_tokens if max_tokensmodel_params = normalize_model_params(model_params)- orig_blk = blkif block_given? && disable_streaming?- result =- perform_completion!(+ return(+ replay_non_streaming_as_streaming!(dialect,user,model_params,@@ -147,273 +162,314 @@ module DiscourseAipartial_tool_calls: partial_tool_calls,output_thinking: output_thinking,cancel_manager: cancel_manager,- execution_context:,+ execution_context: execution_context,+ &blk)-- wrapped = result- wrapped = [result] if !result.is_a?(Array)- wrapped.each do |partial|- blk.call(partial)- break if cancel_manager&.cancelled?- end- return result+ )end@streaming_mode = block_given?prompt = dialect.translate@request_has_images = prompt_has_images?(prompt)+ perform_completion_request_with_retries(+ prompt: prompt,+ dialect: dialect,+ user: user,+ model_params: model_params,+ feature_name: feature_name,+ feature_context: feature_context,+ partial_tool_calls: partial_tool_calls,+ orig_blk: blk,+ cancel_manager: cancel_manager,+ execution_context: execution_context,+ )+ end- structured_output = nil-- if model_params[:response_format].present?- schema_properties =- model_params[:response_format].dig(:json_schema, :schema, :properties)+ def replay_non_streaming_as_streaming!(+ dialect,+ user,+ model_params,+ feature_name:,+ feature_context:,+ partial_tool_calls:,+ output_thinking:,+ cancel_manager:,+ execution_context:,+ &blk+ )+ result =+ perform_completion!(+ dialect,+ user,+ model_params,+ feature_name: feature_name,+ feature_context: feature_context,+ partial_tool_calls: partial_tool_calls,+ output_thinking: output_thinking,+ cancel_manager: cancel_manager,+ execution_context: execution_context,+ )- if schema_properties.present?- structured_output = DiscourseAi::Completions::StructuredOutput.new(schema_properties)- end+ wrapped = result+ wrapped = [result] if !result.is_a?(Array)+ wrapped.each do |partial|+ blk.call(partial)+ break if cancel_manager&.cancelled?end+ result+ end++ def build_structured_output(model_params)+ return if model_params[:response_format].blank?++ schema_properties = model_params[:response_format].dig(:json_schema, :schema, :properties)+ return if schema_properties.blank?- cancel_manager_callback = nil+ DiscourseAi::Completions::StructuredOutput.new(schema_properties)+ end++ def perform_completion_request_with_retries(+ prompt:,+ dialect:,+ user:,+ model_params:,+ feature_name:,+ feature_context:,+ partial_tool_calls:,+ orig_blk:,+ cancel_manager:,+ execution_context:+ )+ request_started_at = Time.nowcancelled = falsecall_status = :error+ retry_count_429 = 0+ # 408/409/5xx and network errors share one transient budget.+ retry_count_transient = 0+ request_attempts = []+ retried = false+ next_attempt_delay_ms = 0+ @forced_json_through_prefill = false+ request_body = prepare_payload(prompt, model_params, dialect).to_json+ log =+ start_completion_log(+ request_body: request_body,+ dialect: dialect,+ prompt: prompt,+ user: user,+ feature_name: feature_name,+ feature_context: feature_context,+ )- FinalDestination::HTTP.start(- model_uri.host,- model_uri.port,- use_ssl: use_ssl?,- read_timeout: TIMEOUT,- open_timeout: TIMEOUT,- write_timeout: TIMEOUT,- ) do |http|- if cancel_manager- cancel_manager_callback =- lambda do- cancelled = true- call_status = :cancelled- http.finish- end- cancel_manager.add_callback(cancel_manager_callback)- end+ loop do+ call_status = :errorresponse_data = +""response_raw = +""# Needed to response token calculations. Cannot rely on response_data due to function buffering.partials_raw = +""- request_body = prepare_payload(prompt, model_params, dialect).to_json+ structured_output = build_structured_output(model_params)request = prepare_request(request_body)-- # Some providers rely on prefill to return structured outputs, so the start- # of the JSON won't be included in the response. Supply it to keep JSON valid.- structured_output << +"{" if structured_output && @forced_json_through_prefill-- log = nil- start_logging =- lambda do- log =- start_log(- provider_id: provider_id,- request_body: request_body,- dialect: dialect,- prompt: prompt,- user: user,- feature_name: feature_name,- feature_context: feature_context,- )- end-- start_logging.call+ retrying = false+ retry_delay = nil+ retry_status = nil+ current_attempt_delay_ms = next_attempt_delay_ms+ next_attempt_delay_ms = 0+ response_output_started = false+ cancel_manager_callback = nilif cancelled || cancel_manager&.cancelled?call_status = :cancelledbreakend- http.request(request) do |response|- log.response_status = response.code.to_i if log+ begin+ FinalDestination::HTTP.start(+ model_uri.host,+ model_uri.port,+ use_ssl: use_ssl?,+ read_timeout: TIMEOUT,+ open_timeout: TIMEOUT,+ write_timeout: TIMEOUT,+ ) do |http|+ if cancel_manager+ cancel_manager_callback =+ lambda do+ cancelled = true+ call_status = :cancelled+ http.finish+ end+ cancel_manager.add_callback(cancel_manager_callback)+ end- if response.code.to_i != 200- Rails.logger.error(- "#{self.class.name}: status: #{response.code.to_i} - body: #{response.body}",- )- response_raw << response.body.to_s- raise CompletionFailed, response.body- end+ begin+ http.request(request) do |response|+ log.response_status = response.code.to_i if log++ if response.code.to_i != 200+ retry_status, retry_delay =+ failed_response_retry_status_and_delay(+ response,+ response_raw,+ retry_count_429: retry_count_429,+ retry_count_transient: retry_count_transient,+ )+ retrying = !retry_delay.nil?+ raise CompletionFailed, response.body+ end++ # Some providers rely on prefill to return structured outputs, so the start+ # of the JSON won't be included in the response. Supply it to keep JSON valid.+ structured_output << +"{" if structured_output && @forced_json_through_prefill- xml_tool_processor =- XmlToolProcessor.new(- partial_tool_calls: partial_tool_calls,- tool_definitions: dialect.prompt.tools,- ) if xml_tools_enabled? && dialect.prompt.has_tools?-- to_strip = xml_tags_to_strip(dialect)- xml_stripper =- DiscourseAi::Completions::XmlTagStripper.new(to_strip) if to_strip.present?-- if @streaming_mode- blk =- lambda do |partial|- if partial.is_a?(String)- partial = xml_stripper << partial if xml_stripper && !partial.empty?-- if structured_output.present?- structured_output << partial if !partial.empty?- partial = structured_output- end+ xml_tool_processor = build_xml_tool_processor(dialect, partial_tool_calls)+ xml_stripper = build_xml_stripper(dialect)++ if @streaming_mode+ blk = streaming_partial_handler(orig_blk, xml_stripper, structured_output)end- orig_blk.call(partial) if partial- end- end- if !@streaming_mode- response_data =- non_streaming_response(- response: response,- xml_tool_processor: xml_tool_processor,- xml_stripper: xml_stripper,- partials_raw: partials_raw,- response_raw: response_raw,- structured_output: structured_output,- )- call_status = :success- return response_data- end+ if !@streaming_mode+ response_data =+ non_streaming_response(+ response: response,+ xml_tool_processor: xml_tool_processor,+ xml_stripper: xml_stripper,+ partials_raw: partials_raw,+ response_raw: response_raw,+ structured_output: structured_output,+ )+ call_status = :success+ return response_data+ end- response.read_body do |chunk|- break if cancelled+ response_data =+ streaming_response(+ response: response,+ blk: blk,+ xml_tool_processor: xml_tool_processor,+ xml_stripper: xml_stripper,+ partials_raw: partials_raw,+ response_raw: response_raw,+ structured_output: structured_output,+ cancelled: -> { cancelled },+ on_output_started: -> { response_output_started = true },+ )+ call_status = :success+ return response_data+ end+ rescue *RETRIABLE_NETWORK_ERRORS => e+ raise if cancelled- response_raw << chunk+ Rails.logger.warn(+ "#{self.class.name}: retryable network error: #{e.class}: #{e.message}",+ )+ if response_output_started+ retrying = false+ raise CompletionFailed, e.message+ end- decode_chunk(chunk).each do |partial|- break if cancelled- partials_raw << partial.to_s- response_data << partial if partial.is_a?(String)- partials = [partial]- if xml_tool_processor && partial.is_a?(String)- partials = (xml_tool_processor << partial)- break if xml_tool_processor.should_cancel?+ retry_status = NETWORK_ERROR_RETRY_STATUS+ retry_delay =+ retry_delay_for_network_error(retry_count_transient: retry_count_transient)+ retrying = !retry_delay.nil?+ raise CompletionFailed, e.message+ ensure+ if cancel_manager && cancel_manager_callback+ cancel_manager.remove_callback(cancel_manager_callback)end- partials.each { |inner_partial| blk.call(inner_partial) }endend+ rescue *RETRIABLE_NETWORK_ERRORS => e+ raise if cancelled- if xml_stripper- stripped = xml_stripper.finish- if stripped.present?- response_data << stripped- result = []- result = (xml_tool_processor << stripped) if xml_tool_processor- result.each { |partial| blk.call(partial) }- end+ Rails.logger.warn(+ "#{self.class.name}: retryable network error: #{e.class}: #{e.message}",+ )+ retry_status = NETWORK_ERROR_RETRY_STATUS+ retry_delay =+ retry_delay_for_network_error(retry_count_transient: retry_count_transient)+ retrying = !retry_delay.nil?++ if retrying+ request_attempts << request_attempt(retry_status, current_attempt_delay_ms)+ retried = true+ next_attempt_delay_ms = retry_delay_to_ms(retry_delay)+ retry_count_transient += 1+ sleep_before_retry(retry_delay, cancel_manager) if retry_delay.positive?+ nextend- xml_tool_processor.finish.each { |partial| blk.call(partial) } if xml_tool_processor- decode_chunk_finish.each { |partial| blk.call(partial) }-- if structured_output- structured_output.finish- if structured_output.broken?- # signal last partial output which will get parsed- # by best effort json parser- blk.call("")++ raise CompletionFailed, e.message+ rescue CompletionFailed+ if retrying && !cancelled+ if retry_status+ request_attempts << request_attempt(retry_status, current_attempt_delay_ms)+ retried = true+ next_attempt_delay_ms = retry_delay_to_ms(retry_delay)+ end++ if retry_status == 429+ retry_count_429 += 1else- # got to signal the end of structured output- blk.call(structured_output)+ retry_count_transient += 1end++ sleep_before_retry(retry_delay, cancel_manager) if retry_delay.positive?+ nextend- call_status = :success- return response_data++ raiseensure- should_log = log && call_status != :cancelled+ should_log = log && call_status != :cancelled && !retryingif should_log- log.raw_response_payload = response_raw- final_log_update(log)- log.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?- log.response_status ||= 200- log.created_at = start_time- log.updated_at = Time.now- log.duration_msecs = (Time.now - start_time) * 1000- log.save!-- execution_context&.token_usage_tracker&.add_from_audit_log(log)-- AiApiRequestStat.record_from_audit_log(log, llm_model: @llm_model)- LlmQuota.log_usage(@llm_model, user, log.request_tokens, log.response_tokens)- LlmCreditAllocation.deduct_credits!(- @llm_model,- feature_name,- log.request_tokens,- log.response_tokens,- )+ if retried+ final_attempt_status = retry_status || log.response_status+ if final_attempt_status+ request_attempts << request_attempt(+ final_attempt_status,+ current_attempt_delay_ms,+ )+ end+ end- # Record Prometheus metrics- DiscourseAi::Completions::LlmMetric.record(- llm_model: @llm_model,+ persist_completion_log!(+ log,+ response_raw: response_raw,+ partials_raw: partials_raw,+ request_attempts: request_attempts.presence,+ call_status: call_status,+ start_time: request_started_at,feature_name: feature_name,- request_tokens: log.request_tokens || 0,- response_tokens: log.response_tokens || 0,- duration_ms: log.duration_msecs,- status: call_status,+ user: user,+ execution_context: execution_context,)-- if Rails.env.development? && ENV["DISCOURSE_AI_DEBUG"]- puts "#{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"- endend- track_failures(call_status)-- if should_log && (logger = execution_context&.audit_logger)- call_data = <<~LOG- #{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}- request:- #{format_possible_json_payload(log.raw_request_payload)}- response:- #{response_data}- LOG- logger.info(call_data)- end- if should_log && (structured_logger = execution_context&.structured_audit_logger)- llm_request =- begin- JSON.parse(log.raw_request_payload)- rescue StandardError- log.raw_request_payload- end+ track_failures(call_status) if !retrying- # gemini puts passwords in query params- # we don't want to log that- llm_call_step = structured_logger.add_child_step(name: "Performing LLM call")-- structured_logger.append_entry(- step: llm_call_step,- name: "llm_call",- args: {- class: self.class.name,- completion_url: request.uri.to_s.split("?")[0],- request: llm_request,- result: response_data,- request_tokens: log.request_tokens,- response_tokens: log.response_tokens,- duration: log.duration_msecs,- stream: @streaming_mode,- },- started_at: start_time.utc,- ended_at: Time.now.utc,+ if should_log+ log_completion_audit_entries(+ log,+ request: request,+ response_data: response_data,+ start_time: request_started_at,+ execution_context: execution_context,)endendendrescue IOError, StandardErrorraise if !cancelled- ensure- if cancel_manager && cancel_manager_callback- cancel_manager.remove_callback(cancel_manager_callback)- endend+ private :replay_non_streaming_as_streaming!,+ :build_structured_output,+ :perform_completion_request_with_retries+def final_log_update(log)# for people that need to overrideend@@ -525,6 +581,357 @@ module DiscourseAiprivate+ def start_completion_log(+ request_body:,+ dialect:,+ prompt:,+ user:,+ feature_name:,+ feature_context:+ )+ start_log(+ provider_id: provider_id,+ request_body: request_body,+ dialect: dialect,+ prompt: prompt,+ user: user,+ feature_name: feature_name,+ feature_context: feature_context,+ )+ end++ # Populated only once retrying occurs. Each entry represents an issued request in+ # the retried sequence, and delay_ms is the planned wait before that request.+ def request_attempt(status, delay_ms)+ { "status" => status, "delay_ms" => delay_ms }+ end++ def retry_delay_to_ms(retry_delay)+ (retry_delay.to_f * 1000).round+ end++ def failed_response_retry_status_and_delay(+ response,+ response_raw,+ retry_count_429:,+ retry_count_transient:+ )+ status = response.code.to_i+ retry_delay =+ retry_delay_for_response(+ response,+ retry_count_429: retry_count_429,+ retry_count_transient: retry_count_transient,+ )++ log_method = retry_delay ? :warn : :error+ Rails.logger.public_send(+ log_method,+ "#{self.class.name}: status: #{status} - body: #{response.body}",+ )+ response_raw << response.body.to_s++ [status, retry_delay]+ end++ def build_xml_tool_processor(dialect, partial_tool_calls)+ return if !xml_tools_enabled? || !dialect.prompt.has_tools?++ XmlToolProcessor.new(+ partial_tool_calls: partial_tool_calls,+ tool_definitions: dialect.prompt.tools,+ )+ end++ def build_xml_stripper(dialect)+ to_strip = xml_tags_to_strip(dialect)+ return if to_strip.blank?++ DiscourseAi::Completions::XmlTagStripper.new(to_strip)+ end++ def streaming_partial_handler(orig_blk, xml_stripper, structured_output)+ lambda do |partial|+ if partial.is_a?(String)+ partial = xml_stripper << partial if xml_stripper && !partial.empty?++ if structured_output.present?+ structured_output << partial if !partial.empty?+ partial = structured_output+ end+ end+ orig_blk.call(partial) if partial+ end+ end++ def streaming_response(+ response:,+ blk:,+ xml_tool_processor:,+ xml_stripper:,+ partials_raw:,+ response_raw:,+ structured_output:,+ cancelled:,+ on_output_started:+ )+ response_data = +""++ response.read_body do |chunk|+ break if cancelled.call++ response_raw << chunk++ decode_chunk(chunk).each do |partial|+ break if cancelled.call+ partials_raw << partial.to_s+ response_data << partial if partial.is_a?(String)+ partials = [partial]+ if xml_tool_processor && partial.is_a?(String)+ partials = (xml_tool_processor << partial)+ break if xml_tool_processor.should_cancel?+ end+ partials.each do |inner_partial|+ on_output_started.call+ blk.call(inner_partial)+ end+ end+ end++ finish_streaming_response(+ response_data: response_data,+ blk: blk,+ xml_tool_processor: xml_tool_processor,+ xml_stripper: xml_stripper,+ structured_output: structured_output,+ )++ response_data+ end++ def finish_streaming_response(+ response_data:,+ blk:,+ xml_tool_processor:,+ xml_stripper:,+ structured_output:+ )+ if xml_stripper+ stripped = xml_stripper.finish+ if stripped.present?+ response_data << stripped+ result = []+ result = (xml_tool_processor << stripped) if xml_tool_processor+ result.each { |partial| blk.call(partial) }+ end+ end++ xml_tool_processor.finish.each { |partial| blk.call(partial) } if xml_tool_processor+ decode_chunk_finish.each { |partial| blk.call(partial) }++ finish_structured_output_streaming(structured_output, blk) if structured_output+ end++ def finish_structured_output_streaming(structured_output, blk)+ structured_output.finish+ if structured_output.broken?+ # signal last partial output which will get parsed+ # by best effort json parser+ blk.call("")+ else+ # got to signal the end of structured output+ blk.call(structured_output)+ end+ end++ def persist_completion_log!(+ log,+ response_raw:,+ partials_raw:,+ request_attempts:,+ call_status:,+ start_time:,+ feature_name:,+ user:,+ execution_context:+ )+ log.raw_response_payload = response_raw+ log.request_attempts = request_attempts if log.has_attribute?(:request_attempts)+ final_log_update(log)+ log.response_tokens = tokenizer.size(partials_raw) if log.response_tokens.blank?+ log.response_status ||= 200 if call_status == :success+ log.created_at = start_time+ log.updated_at = Time.now+ log.duration_msecs = (Time.now - start_time) * 1000+ log.save!++ execution_context&.token_usage_tracker&.add_from_audit_log(log)++ AiApiRequestStat.record_from_audit_log(log, llm_model: @llm_model)+ LlmQuota.log_usage(@llm_model, user, log.request_tokens, log.response_tokens)+ LlmCreditAllocation.deduct_credits!(+ @llm_model,+ feature_name,+ log.request_tokens,+ log.response_tokens,+ )++ DiscourseAi::Completions::LlmMetric.record(+ llm_model: @llm_model,+ feature_name: feature_name,+ request_tokens: log.request_tokens || 0,+ response_tokens: log.response_tokens || 0,+ duration_ms: log.duration_msecs,+ status: call_status,+ )++ if Rails.env.development? && ENV["DISCOURSE_AI_DEBUG"]+ puts "#{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}"+ end+ end++ def log_completion_audit_entries(+ log,+ request:,+ response_data:,+ start_time:,+ execution_context:+ )+ log_text_audit_entry(log, response_data, execution_context&.audit_logger)+ log_structured_audit_entry(+ log,+ request: request,+ response_data: response_data,+ start_time: start_time,+ structured_logger: execution_context&.structured_audit_logger,+ )+ end++ def log_text_audit_entry(log, response_data, logger)+ return if !logger++ call_data = <<~LOG+ #{self.class.name}: request_tokens #{log.request_tokens} response_tokens #{log.response_tokens}+ request:+ #{format_possible_json_payload(log.raw_request_payload)}+ response:+ #{response_data}+ LOG+ logger.info(call_data)+ end++ def log_structured_audit_entry(+ log,+ request:,+ response_data:,+ start_time:,+ structured_logger:+ )+ return if !structured_logger++ llm_request =+ begin+ JSON.parse(log.raw_request_payload)+ rescue StandardError+ log.raw_request_payload+ end++ # gemini puts passwords in query params+ # we don't want to log that+ llm_call_step = structured_logger.add_child_step(name: "Performing LLM call")++ structured_logger.append_entry(+ step: llm_call_step,+ name: "llm_call",+ args: {+ class: self.class.name,+ completion_url: request.uri.to_s.split("?")[0],+ request: llm_request,+ result: response_data,+ request_tokens: log.request_tokens,+ response_tokens: log.response_tokens,+ duration: log.duration_msecs,+ stream: @streaming_mode,+ },+ started_at: start_time.utc,+ ended_at: Time.now.utc,+ )+ end++ def retry_delay_for_response(response, retry_count_429:, retry_count_transient:)+ status = response.code.to_i++ if status == 429 && retry_count_429 < RATE_LIMIT_RETRY_DELAYS.length+ delay = RATE_LIMIT_RETRY_DELAYS[retry_count_429]+ retry_after_delay = [+ retry_after_delay(response["Retry-After"]),+ retry_delay_from_response_body(response.body),+ ].compact.max++ return retry_delay_with_jitter([delay, retry_after_delay].compact.max)+ end++ if transient_error_status?(status) &&+ retry_count_transient < TRANSIENT_ERROR_RETRY_DELAYS.length+ delay = TRANSIENT_ERROR_RETRY_DELAYS[retry_count_transient]+ retry_after_delay = [+ retry_after_delay(response["Retry-After"]),+ retry_delay_from_response_body(response.body),+ ].compact.max++ return retry_delay_with_jitter([delay, retry_after_delay].compact.max)+ end++ nil+ end++ def retry_delay_for_network_error(retry_count_transient:)+ return if retry_count_transient >= TRANSIENT_ERROR_RETRY_DELAYS.length++ TRANSIENT_ERROR_RETRY_DELAYS[retry_count_transient] + retry_jitter+ end++ def retry_delay_with_jitter(delay)+ [delay + retry_jitter, MAX_RETRY_AFTER_SECONDS].min+ end++ def transient_error_status?(status)+ status == 408 || status == 409 || (status >= 500 && status < 600)+ end++ def sleep_before_retry(delay, cancel_manager = nil)+ if cancel_manager+ cancel_manager.wait_for_cancel(delay) if !cancel_manager.cancelled?+ else+ sleep(delay)+ end+ end++ def retry_after_delay(header)+ return if header.blank?++ delay =+ if header.to_s.match?(/\A\d+(\.\d+)?\z/)+ header.to_f+ else+ Time.httpdate(header).to_f - Time.now.to_f+ end++ return if delay <= 0++ [delay, MAX_RETRY_AFTER_SECONDS].min+ rescue StandardError+ nil+ end++ def retry_delay_from_response_body(_body)+ nil+ end++ def retry_jitter+ rand * RETRY_JITTER_MAX_SECONDS+ end+def format_possible_json_payload(payload)JSON.pretty_generate(JSON.parse(payload))rescue JSON::ParserErrordiff --git a/plugins/discourse-ai/lib/completions/endpoints/gemini.rb b/plugins/discourse-ai/lib/completions/endpoints/gemini.rbindex 1c0275a0013..d13ef5bf0fd 100644--- a/plugins/discourse-ai/lib/completions/endpoints/gemini.rb+++ b/plugins/discourse-ai/lib/completions/endpoints/gemini.rb@@ -139,15 +139,45 @@ module DiscourseAipayload[:generationConfig][:thinkingConfig][:includeThoughts] = trueend+ payload[:serviceTier] = service_tier if service_tier.present?+payloadend+ def service_tier+ return @service_tier if defined?(@service_tier)++ @service_tier = llm_model.lookup_custom_param("service_tier")+ @service_tier = nil if !%w[standard flex priority].include?(@service_tier)+ @service_tier+ end+def prepare_request(payload)headers = { "Content-Type" => "application/json" }Net::HTTP::Post.new(model_uri, headers).tap { |r| r.body = payload }end+ def retry_delay_from_response_body(body)+ return if body.blank? || body.bytesize >= 10_000++ retry_info =+ Array(JSON.parse(body).dig("error", "details")).find do |detail|+ detail["@type"] == "type.googleapis.com/google.rpc.RetryInfo"+ end++ retry_delay = retry_info&.[]("retryDelay")+ match = retry_delay.to_s.match(/\A(?<seconds>\d+(?:\.\d+)?)s\z/)+ return if !match++ delay = match[:seconds].to_f+ return if delay <= 0++ [delay, MAX_RETRY_AFTER_SECONDS].min+ rescue StandardError+ nil+ end+def extract_completion_from(response_raw)parsed =if @streaming_modediff --git a/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rb b/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rbindex 5d1d8661ced..784ec8d6065 100644--- a/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/cancel_manager_spec.rb@@ -36,6 +36,30 @@ describe DiscourseAi::Completions::CancelManager doexpect(cancel_manager.monitor_thread).to be_nilend+ it "waits until timeout when not cancelled" do+ cancel_manager = DiscourseAi::Completions::CancelManager.new++ expect(cancel_manager.wait_for_cancel(0.001)).to eq(false)+ end++ it "wakes waiters when cancelled" do+ cancel_manager = DiscourseAi::Completions::CancelManager.new+ waiting = Queue.new++ waiter =+ Thread.new do+ waiting << true+ cancel_manager.wait_for_cancel(60)+ end++ waiting.pop+ cancel_manager.cancel!++ expect(waiter.join(1)).to eq(waiter)+ ensure+ waiter&.kill+ end+it "should do nothing when cancel manager is already cancelled" docancel_manager = DiscourseAi::Completions::CancelManager.newcancel_manager.cancel!diff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rbindex 29e17da8f91..8950de19072 100644--- a/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/aws_bedrock_converse_spec.rb@@ -312,6 +312,23 @@ RSpec.describe DiscourseAi::Completions::Endpoints::AwsBedrockConverse doexpect { llm.generate("hello", user: user) }.to raise_error(DiscourseAi::Completions::Endpoints::Base::CompletionFailed,)+ expect(AiApiAuditLog.last.response_status).to be_nil+ end+ it "records SDK error response status when available" do+ client = instance_double(Aws::BedrockRuntime::Client)+ context = Seahorse::Client::RequestContext.new+ context.http_response.status_code = 429+ allow(client).to receive(:converse).and_raise(+ Aws::BedrockRuntime::Errors::ThrottlingException.new(context, "Rate exceeded"),+ )+ allow(Aws::BedrockRuntime::Client).to receive(:new).and_return(client)++ llm = DiscourseAi::Completions::Llm.proxy("custom:#{model.id}")++ expect { llm.generate("hello", user: user) }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ expect(AiApiAuditLog.last.response_status).to eq(429)endendenddiff --git a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rbindex 239a2101249..b3441a5c123 100644--- a/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/endpoints/gemini_spec.rb@@ -188,8 +188,210 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Gemini do}end+ def gemini_rate_limit_body(retry_delay: "45s")+ {+ error: {+ code: 429,+ message: "quota exceeded",+ status: "RESOURCE_EXHAUSTED",+ details: [{ "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: retry_delay }],+ },+ }.to_json+ end+before { enable_current_plugin }+ it "uses Gemini RetryInfo retryDelay from rate limit response bodies" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(45, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ request =+ stub_request(:post, url).to_return(+ { status: 429, body: gemini_rate_limit_body(retry_delay: "45s") },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ expect(request).to have_been_requested.times(2)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 45_000 }],+ )+ end++ it "caps Gemini RetryInfo retryDelay from rate limit response bodies" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(60, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ stub_request(:post, url).to_return(+ { status: 429, body: gemini_rate_limit_body(retry_delay: "120s") },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end++ it "ignores oversized Gemini rate limit response bodies when extracting retry delays" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(2, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ stub_request(:post, url).to_return(+ { status: 429, body: gemini_rate_limit_body(retry_delay: "45s") + (" " * 10_000) },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end++ it "uses the larger retry delay from Gemini RetryInfo and Retry-After" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(10, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ stub_request(:post, url).to_return(+ {+ status: 429,+ body: gemini_rate_limit_body(retry_delay: "10s"),+ headers: {+ "Retry-After" => "5",+ },+ },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end++ it "uses Gemini RetryInfo retryDelay for transient errors" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(12, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ stub_request(:post, url).to_return(+ { status: 503, body: gemini_rate_limit_body(retry_delay: "12s") },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end++ it "supports fractional Gemini RetryInfo retryDelay values" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(2.5, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ stub_request(:post, url).to_return(+ { status: 429, body: gemini_rate_limit_body(retry_delay: "2.5s") },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end++ it "ignores non-positive Gemini RetryInfo retryDelay values" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(2, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ stub_request(:post, url).to_return(+ { status: 429, body: gemini_rate_limit_body(retry_delay: "0s") },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end++ it "finds Gemini RetryInfo among other error details" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(7, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ body = {+ error: {+ code: 429,+ details: [+ { "@type": "type.googleapis.com/google.rpc.Help" },+ { "@type": "type.googleapis.com/google.rpc.RetryInfo", retryDelay: "7s" },+ ],+ },+ }.to_json+ stub_request(:post, url).to_return(+ { status: 429, body: body },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end++ it "ignores invalid Gemini rate limit response bodies when extracting retry delays" do+ DiscourseAi::Completions::Endpoints::Gemini.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Gemini+ .any_instance+ .expects(:sleep_before_retry)+ .with(2, nil)+ .once++ url = "#{model.url}:generateContent?key=123"+ stub_request(:post, url).to_return(+ { status: 429, body: "{" },+ { status: 200, body: gemini_mock.response("ok").to_json },+ )++ llm = DiscourseAi::Completions::Llm.proxy(model)++ expect(llm.generate("Hello", user: user)).to eq("ok")+ end+it "correctly configures thinking when enabled" domodel.update!(provider_params: { enable_thinking: "true", thinking_tokens: "10000" })@@ -476,6 +678,81 @@ RSpec.describe DiscourseAi::Completions::Endpoints::Gemini doexpect(parsed.dig(:generationConfig, :thinkingConfig)).to be_nilend+ it "adds configured service tier using the Gemini API field name" do+ response = gemini_mock.response("Configured service tier").to_json+ url = "#{model.url}:generateContent?key=123"++ captured_bodies = []+ stub_request(:post, url).with(+ body:+ proc do |req_body|+ captured_bodies << JSON.parse(req_body, symbolize_names: true)+ true+ end,+ ).to_return(status: 200, body: response)++ {+ "default" => nil,+ "standard" => "standard",+ "flex" => "flex",+ "priority" => "priority",+ }.each do |configured_tier, expected_tier|+ model.update!(provider_params: { service_tier: configured_tier })++ DiscourseAi::Completions::Llm.proxy(model).generate("Hello", user: user)++ payload = captured_bodies.last+ if expected_tier+ expect(payload[:serviceTier]).to eq(expected_tier)+ else+ expect(payload).not_to have_key(:serviceTier)+ end+ expect(payload).not_to have_key(:service_tier)+ end+ end++ it "omits service tier when it is not configured" do+ response = gemini_mock.response("No service tier").to_json+ req_body = nil++ llm = DiscourseAi::Completions::Llm.proxy(model)+ url = "#{model.url}:generateContent?key=123"++ stub_request(:post, url).with(+ body:+ proc do |_req_body|+ req_body = _req_body+ true+ end,+ ).to_return(status: 200, body: response)++ llm.generate("Hello", user: user)++ expect(JSON.parse(req_body, symbolize_names: true)).not_to have_key(:serviceTier)+ end++ it "omits service tier when it is invalid" do+ model.update!(provider_params: { service_tier: "invalid" })++ response = gemini_mock.response("Invalid service tier").to_json+ req_body = nil++ llm = DiscourseAi::Completions::Llm.proxy(model)+ url = "#{model.url}:generateContent?key=123"++ stub_request(:post, url).with(+ body:+ proc do |_req_body|+ req_body = _req_body+ true+ end,+ ).to_return(status: 200, body: response)++ llm.generate("Hello", user: user)++ expect(JSON.parse(req_body, symbolize_names: true)).not_to have_key(:serviceTier)+ end+it "strips temperature when thinking_level is set" domodel.update!(provider_params: { enable_thinking: true, thinking_level: "high" })diff --git a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb b/plugins/discourse-ai/spec/lib/completions/llm_spec.rbindex ea6865aaf66..e25aadbdd2b 100644--- a/plugins/discourse-ai/spec/lib/completions/llm_spec.rb+++ b/plugins/discourse-ai/spec/lib/completions/llm_spec.rb@@ -29,7 +29,7 @@ RSpec.describe DiscourseAi::Completions::Llm dodef streaming_body(content: "Hello")<<~SSE- data: {"id":"1","object":"chat.completion.chunk","choices":[{"delta":{"content":"#{content}"}}]}+ data: {"id":"1","object":"chat.completion.chunk","choices":[{"delta":{"content":#{content.to_json}}}]}data: [DONE]SSE@@ -63,6 +63,17 @@ RSpec.describe DiscourseAi::Completions::Llm dollm.generate("hi", user:) { |partial| result << partial }expect(result).to eq("Hi")end++ it "replays non-streaming responses when streaming is disabled" do+ model.update!(provider_params: { "disable_streaming" => true })+ stub_response(body: success_body(content: "Hi"))++ partials = []+ result = llm.generate("hi", user:) { |partial| partials << partial }++ expect(result).to eq("Hi")+ expect(partials).to eq(["Hi"])+ endendcontext "with a fake model" do@@ -87,6 +98,33 @@ RSpec.describe DiscourseAi::Completions::Llm doendend+ context "with structured output" do+ it "returns a structured output buffer" do+ stub_response(body: success_body(content: '{"message":"ok"}'))++ result =+ llm.generate(+ "hello",+ user:,+ response_format: {+ json_schema: {+ schema: {+ properties: {+ message: {+ type: "string",+ },+ },+ },+ },+ },+ )++ expect(result).to be_a(DiscourseAi::Completions::StructuredOutput)+ expect(result).to be_finished+ expect(result.to_s).to eq('{"message":"ok"}')+ end+ end+context "when auditing" doit "logs topic_id, post_id, feature_name, and feature_context" dostub_response(body: success_body)@@ -169,9 +207,436 @@ RSpec.describe DiscourseAi::Completions::Llm doendend+ context "when retrying failed requests" do+ before do+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:sleep_before_retry)+ end++ it "retries rate limits three times" do+ request =+ WebMock.stub_request(:post, model.url).to_return(+ { status: 429, body: "rate limited" },+ { status: 429, body: "rate limited" },+ { status: 429, body: "rate limited" },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ result = nil++ expect { result = llm.generate("Hello", user:) }.to change { AiApiAuditLog.count }.by(1)++ expect(result).to eq("ok")+ expect(request).to have_been_requested.times(4)+ expect(AiApiAuditLog.last.response_status).to eq(200)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [+ { "status" => 429, "delay_ms" => 0 },+ { "status" => 429, "delay_ms" => 2000 },+ { "status" => 429, "delay_ms" => 8000 },+ { "status" => 200, "delay_ms" => 16_000 },+ ],+ )+ end++ it "does not retry non-retryable client errors" do+ request =+ WebMock.stub_request(:post, model.url).to_return(status: 401, body: "unauthorized")++ expect { llm.generate("Hello", user:) }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ expect(request).to have_been_requested.once+ expect(AiApiAuditLog.last.response_status).to eq(401)+ expect(AiApiAuditLog.last.request_attempts).to be_nil+ end++ it "includes retry waits in audit duration" do+ start_time = Time.utc(2026, 1, 1, 12, 0, 0)+ current_time = start_time+ Time.stubs(:now).returns(current_time)+ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:sleep_before_retry)+ .with do |delay, cancel_manager|+ current_time += delay.seconds+ Time.stubs(:now).returns(current_time)+ delay == 2 && cancel_manager.nil?+ end+ .once++ WebMock.stub_request(:post, model.url).to_return(+ { status: 429, body: "rate limited" },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],+ )+ expect(AiApiAuditLog.last.duration_msecs).to be >= 2000+ end++ it "respects retry-after for rate limits" do+ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:sleep_before_retry)+ .with(5, nil)+ .once++ WebMock.stub_request(:post, model.url).to_return(+ { status: 429, body: "rate limited", headers: { "Retry-After" => "5" } },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ end++ it "respects retry-after HTTP dates for rate limits" do+ freeze_time++ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:sleep_before_retry)+ .with { |delay, cancel_manager| delay.between?(29, 30) && cancel_manager.nil? }+ .once++ WebMock.stub_request(:post, model.url).to_return(+ {+ status: 429,+ body: "rate limited",+ headers: {+ "Retry-After" => 30.seconds.from_now.httpdate,+ },+ },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ end++ it "caps retry-after values after adding jitter" do+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:retry_jitter).returns(1)+ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:sleep_before_retry)+ .with(60, nil)+ .once++ WebMock.stub_request(:post, model.url).to_return(+ { status: 429, body: "rate limited", headers: { "Retry-After" => "999999" } },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ end++ it "raises rate limits after three retries" do+ request =+ WebMock.stub_request(:post, model.url).to_return(status: 429, body: "rate limited")++ expect { llm.generate("Hello", user:) }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ expect(request).to have_been_requested.times(4)+ expect(AiApiAuditLog.last.response_status).to eq(429)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [+ { "status" => 429, "delay_ms" => 0 },+ { "status" => 429, "delay_ms" => 2000 },+ { "status" => 429, "delay_ms" => 8000 },+ { "status" => 429, "delay_ms" => 16_000 },+ ],+ )+ end++ it "retries streaming responses after rate limits" do+ request =+ WebMock.stub_request(:post, model.url).to_return(+ { status: 429, body: "rate limited" },+ { status: 200, body: streaming_body(content: "Hi") },+ )++ result = +""+ llm.generate("Hello", user:) { |partial| result << partial }++ expect(result).to eq("Hi")+ expect(request).to have_been_requested.times(2)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],+ )+ end++ it "does not retry streaming responses after output has started" do+ request =+ WebMock.stub_request(:post, model.url).to_return(status: 200, body: streaming_body)++ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:streaming_response)+ .with do |kwargs|+ kwargs[:on_output_started].call+ kwargs[:blk].call("partial")+ true+ end+ .raises(Net::ReadTimeout.new("timed out"))++ result = +""+ expect { llm.generate("Hello", user:) { |partial| result << partial } }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ expect(result).to eq("partial")+ expect(request).to have_been_requested.once+ expect(AiApiAuditLog.last.response_status).to eq(200)+ expect(AiApiAuditLog.last.request_attempts).to be_nil+ end++ it "retries streaming structured output after rate limits" do+ request =+ WebMock.stub_request(:post, model.url).to_return(+ { status: 429, body: "rate limited" },+ { status: 200, body: streaming_body(content: '{"message":"ok"}') },+ )++ result = nil+ llm.generate(+ "Hello",+ user:,+ response_format: {+ json_schema: {+ schema: {+ properties: {+ message: {+ type: "string",+ },+ },+ },+ },+ },+ ) { |partial| result = partial }++ expect(result).to be_a(DiscourseAi::Completions::StructuredOutput)+ expect(result.to_s).to eq('{"message":"ok"}')+ expect(request).to have_been_requested.times(2)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],+ )+ end++ it "returns structured output after rate limits" do+ WebMock.stub_request(:post, model.url).to_return(+ { status: 429, body: "rate limited" },+ { status: 200, body: success_body(content: '{"message":"ok"}').to_json },+ )++ result =+ llm.generate(+ "Hello",+ user:,+ response_format: {+ json_schema: {+ schema: {+ properties: {+ message: {+ type: "string",+ },+ },+ },+ },+ },+ )++ expect(result).to be_a(DiscourseAi::Completions::StructuredOutput)+ expect(result.to_s).to eq('{"message":"ok"}')+ end++ it "retries network errors twice" do+ request =+ WebMock+ .stub_request(:post, model.url)+ .to_raise(Net::ReadTimeout.new("timed out"))+ .then+ .to_raise(Errno::ECONNRESET.new)+ .then+ .to_return(status: 200, body: success_body(content: "ok").to_json)++ expect(llm.generate("Hello", user:)).to eq("ok")+ expect(request).to have_been_requested.times(3)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [+ { "status" => 0, "delay_ms" => 0 },+ { "status" => 0, "delay_ms" => 500 },+ { "status" => 200, "delay_ms" => 1000 },+ ],+ )+ end++ it "raises network errors after two retries" do+ request = WebMock.stub_request(:post, model.url).to_raise(Net::ReadTimeout.new("timed out"))++ expect { llm.generate("Hello", user:) }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ expect(request).to have_been_requested.times(3)+ expect(AiApiAuditLog.last.response_status).to be_nil+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [+ { "status" => 0, "delay_ms" => 0 },+ { "status" => 0, "delay_ms" => 500 },+ { "status" => 0, "delay_ms" => 1000 },+ ],+ )+ end++ it "retries request timeouts twice" do+ request =+ WebMock.stub_request(:post, model.url).to_return(+ { status: 408, body: "timeout" },+ { status: 408, body: "timeout" },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ expect(request).to have_been_requested.times(3)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [+ { "status" => 408, "delay_ms" => 0 },+ { "status" => 408, "delay_ms" => 500 },+ { "status" => 200, "delay_ms" => 1000 },+ ],+ )+ end++ it "retries lock timeouts twice" do+ request =+ WebMock.stub_request(:post, model.url).to_return(+ { status: 409, body: "conflict" },+ { status: 409, body: "conflict" },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ expect(request).to have_been_requested.times(3)+ end++ it "retries server errors twice" do+ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:sleep_before_retry)+ .with(0.5, nil)+ .once+ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:sleep_before_retry)+ .with(1.0, nil)+ .once++ request =+ WebMock.stub_request(:post, model.url).to_return(+ { status: 503, body: "unavailable" },+ { status: 503, body: "unavailable" },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ expect(request).to have_been_requested.times(3)+ end++ it "respects retry-after for server errors" do+ DiscourseAi::Completions::Endpoints::Base+ .any_instance+ .expects(:sleep_before_retry)+ .with(5, nil)+ .once++ WebMock.stub_request(:post, model.url).to_return(+ { status: 503, body: "unavailable", headers: { "Retry-After" => "5" } },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [{ "status" => 503, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 5000 }],+ )+ end++ it "tracks mixed request attempts" do+ request =+ WebMock.stub_request(:post, model.url).to_return(+ { status: 503, body: "unavailable" },+ { status: 429, body: "rate limited" },+ { status: 200, body: success_body(content: "ok").to_json },+ )++ expect(llm.generate("Hello", user:)).to eq("ok")+ expect(request).to have_been_requested.times(3)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [+ { "status" => 503, "delay_ms" => 0 },+ { "status" => 429, "delay_ms" => 500 },+ { "status" => 200, "delay_ms" => 2000 },+ ],+ )+ end++ it "raises server errors after two retries" do+ request = WebMock.stub_request(:post, model.url).to_return(status: 503, body: "unavailable")++ expect { llm.generate("Hello", user:) }.to raise_error(+ DiscourseAi::Completions::Endpoints::Base::CompletionFailed,+ )+ expect(request).to have_been_requested.times(3)+ expect(AiApiAuditLog.last.response_status).to eq(503)+ expect(AiApiAuditLog.last.request_attempts).to eq(+ [+ { "status" => 503, "delay_ms" => 0 },+ { "status" => 503, "delay_ms" => 500 },+ { "status" => 503, "delay_ms" => 1000 },+ ],+ )+ end+ end++ context "when sleeping before retries" do+ it "sleeps normally without a cancel manager" do+ endpoint = DiscourseAi::Completions::Endpoints::Base.new(model)+ endpoint.expects(:sleep).with(3).once++ endpoint.send(:sleep_before_retry, 3, nil)+ end++ it "stops when cancelled" do+ cancel_manager = DiscourseAi::Completions::CancelManager.new+ endpoint = DiscourseAi::Completions::Endpoints::Base.new(model)+ waiting = Queue.new++ sleep_thread =+ Thread.new do+ waiting << true+ endpoint.send(:sleep_before_retry, 60, cancel_manager)+ end++ waiting.pop+ cancel_manager.cancel!++ expect(sleep_thread.join(1)).to eq(sleep_thread)+ ensure+ sleep_thread&.kill+ end+ end+context "when tracking failures" do+ before do+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:retry_jitter).returns(0)+ DiscourseAi::Completions::Endpoints::Base.any_instance.stubs(:sleep_before_retry)+ end+it "fast-tracks problem check after threshold and resets on success" doWebMock.stub_request(:post, model.url).to_return(+ { status: 500, body: "fail" },+ { status: 500, body: "fail" },+ { status: 500, body: "fail" },+ { status: 500, body: "fail" },{ status: 500, body: "fail" },{ status: 500, body: "fail" },{ status: 200, body: success_body.to_json },diff --git a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rbindex 21ca7f1c5fe..5146ea3f84e 100644--- a/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/admin/ai_llms_controller_spec.rb@@ -84,6 +84,18 @@ RSpec.describe DiscourseAi::Admin::AiLlmsController doexpect(vllm_params).not_to have_key("enable_thinking")end+ it "includes Gemini service tier metadata" do+ get "/admin/plugins/discourse-ai/ai-llms.json"+ expect(response).to be_successful++ google_params = response.parsed_body.dig("meta", "provider_params", "google")+ expect(google_params["service_tier"]).to include(+ "type" => "enum",+ "values" => %w[default standard flex priority],+ "default" => "default",+ )+ end+it "lists enabled features on appropriate LLMs" doSiteSetting.ai_bot_enabled = trueSiteSetting.ai_bot_enabled_llms = llm_model.id.to_sdiff --git a/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb b/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rbnew file mode 100644index 00000000000..48e13aa3f25--- /dev/null+++ b/plugins/discourse-ai/spec/serializers/ai_api_audit_log_serializer_spec.rb@@ -0,0 +1,36 @@+# frozen_string_literal: true++RSpec.describe AiApiAuditLogSerializer do+ before { enable_current_plugin }++ it "omits request attempts when the column is unavailable" do+ log = AiApiAuditLog.new(provider_id: AiApiAuditLog::Provider::OpenAI)+ original_has_attribute = log.method(:has_attribute?)+ log.define_singleton_method(:has_attribute?) do |attribute|+ attribute.to_sym == :request_attempts ? false : original_has_attribute.call(attribute)+ end++ serialized = described_class.new(log).as_json[:ai_api_audit_log]++ expect(serialized).not_to have_key(:request_attempts)+ end++ it "includes request attempts when the column is available" do+ log =+ AiApiAuditLog.new(+ provider_id: AiApiAuditLog::Provider::OpenAI,+ response_status: 200,+ request_attempts: [+ { "status" => 429, "delay_ms" => 0 },+ { "status" => 200, "delay_ms" => 2000 },+ ],+ )++ serialized = described_class.new(log).as_json[:ai_api_audit_log]++ expect(serialized[:request_attempts]).to eq(+ [{ "status" => 429, "delay_ms" => 0 }, { "status" => 200, "delay_ms" => 2000 }],+ )+ expect(serialized[:response_status]).to eq(200)+ end+end