A platform for community discussion. Free, open, simple.
FIX: Pause failing embedding providers (#42539)
Samfeb3fba6a3bd
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rbindex 8c0951f4369..fbde5e7f9bb 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb@@ -113,8 +113,9 @@ module DiscourseAiDiscourseAi::Embeddings::Vector.new(embedding_def).vector_from("this is a test")render json: { success: true }- rescue Net::HTTPBadResponse => e- render json: { success: false, error: e.message }+ rescue DiscourseAi::Inference::EmbeddingInferenceError,+ DiscourseAi::Embeddings::ProviderPausedError => error+ render json: { success: false, error: error.message }endprivatediff --git a/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rbindex 1f8665ce5b5..bb837d6f866 100644--- a/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rb+++ b/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rb@@ -71,8 +71,8 @@ module DiscourseAi.each { |topic_post| grouped_results.add(topic_post) }rescue Discourse::InvalidAccessreturn render_json_error(I18n.t("invalid_access"), status: 403)- rescue Net::HTTPBadResponse => e- Rails.logger.warn("Semantic search embedding generation failed: #{e.message}")+ rescue DiscourseAi::Inference::EmbeddingInferenceError,+ DiscourseAi::Embeddings::ProviderPausedErrorendrender_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)@@ -110,8 +110,8 @@ module DiscourseAisemantic_search.search_for_topics(query, _page = 1, hyde: false).each { |topic_post| grouped_results.add(topic_post) }- rescue Net::HTTPBadResponse => e- Rails.logger.warn("Quick search embedding generation failed: #{e.message}")+ rescue DiscourseAi::Inference::EmbeddingInferenceError,+ DiscourseAi::Embeddings::ProviderPausedErrorendrender_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)diff --git a/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rb b/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rbindex ab3ea167813..5413b4d72d3 100644--- a/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rb+++ b/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rb@@ -16,7 +16,14 @@ module Jobsreturn if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pmsreturn if post.raw.blank?- DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)+ definition = EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_selected_model)+ return if DiscourseAi::Embeddings::ProviderHealth.paused?(definition)++ DiscourseAi::Embeddings::Vector.new(definition).generate_representation_from(target)+ rescue DiscourseAi::Inference::EmbeddingInferenceError => error+ raise if !error.terminal?+ rescue DiscourseAi::Embeddings::ProviderPausedError+ nilendendenddiff --git a/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rbindex 163534cc240..329119aa7b6 100644--- a/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rb+++ b/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rb@@ -33,6 +33,8 @@ module Jobstopic_work_list << backfill_vector if backfill_vectortopic_work_list.each do |vector|+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)+rebaked = 0table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLEvector_def = vector.vdef@@ -48,6 +50,7 @@ module Jobsrebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)next if rebaked >= limit# Then, we'll try to backfill embeddings for topics that have outdated@@ -60,6 +63,7 @@ module Jobsrebaked += populate_topic_embeddings(vector, relation, force: true)+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)next if rebaked >= limit# Finally, we'll try to backfill embeddings for topics that have outdated@@ -72,6 +76,7 @@ module Jobspopulate_topic_embeddings(vector, relation, force: true)+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)next unless SiteSetting.ai_embeddings_per_post_enabled# Now for postsdiff --git a/plugins/discourse-ai/app/models/embedding_definition.rb b/plugins/discourse-ai/app/models/embedding_definition.rbindex 03c2df8a5ca..19bacd4ec7a 100644--- a/plugins/discourse-ai/app/models/embedding_definition.rb+++ b/plugins/discourse-ai/app/models/embedding_definition.rb@@ -131,11 +131,17 @@ class EmbeddingDefinition < ActiveRecord::Basevalidate :api_key_or_secret_presentafter_create :create_indexes+ after_update_commit :clear_provider_pause+ after_destroy_commit :clear_provider_pausedef create_indexesJobs.enqueue(:manage_embedding_def_search_index, id: id)end+ def clear_provider_pause+ DiscourseAi::Embeddings::ProviderHealth.clear!(id)+ end+def tokenizertokenizer_class.constantizeenddiff --git a/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rb b/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rbindex 84b1c821e84..b245e41ef7d 100644--- a/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rb+++ b/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rb@@ -25,7 +25,8 @@ module DiscourseAiendtrue- rescue Net::HTTPBadResponse+ rescue DiscourseAi::Inference::EmbeddingInferenceError,+ DiscourseAi::Embeddings::ProviderPausedErrorfalseenddiff --git a/plugins/discourse-ai/lib/embeddings/entry_point.rb b/plugins/discourse-ai/lib/embeddings/entry_point.rbindex 1c625ee1577..61b839cf166 100644--- a/plugins/discourse-ai/lib/embeddings/entry_point.rb+++ b/plugins/discourse-ai/lib/embeddings/entry_point.rb@@ -53,7 +53,9 @@ module DiscourseAi# embeddings generation.callback =Proc.new do |target|+ definition = EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_selected_model)if DiscourseAi::Embeddings.enabled? &&+ !DiscourseAi::Embeddings::ProviderHealth.paused?(definition) &&(target.is_a?(Topic) || SiteSetting.ai_embeddings_per_post_enabled)Jobs.enqueue(:generate_embeddings,@@ -77,10 +79,15 @@ module DiscourseAiif DiscourseAi::Embeddings.enabled?query = [args[:title], args[:raw]].join("\n\n")- DiscourseAi::Embeddings::SemanticSearch- .new(args[:guardian])- .similar_topic_ids_to(query, candidates: args[:candidates])- .each { |similar_topic_id| plugin_candidate_ids << similar_topic_id }+ begin+ DiscourseAi::Embeddings::SemanticSearch+ .new(args[:guardian])+ .similar_topic_ids_to(query, candidates: args[:candidates])+ .each { |similar_topic_id| plugin_candidate_ids << similar_topic_id }+ rescue DiscourseAi::Inference::EmbeddingInferenceError,+ DiscourseAi::Embeddings::ProviderPausedError+ # Ordinary similar-topic candidates remain available.+ endendendenddiff --git a/plugins/discourse-ai/lib/embeddings/provider_health.rb b/plugins/discourse-ai/lib/embeddings/provider_health.rbnew file mode 100644index 00000000000..19a3856108f--- /dev/null+++ b/plugins/discourse-ai/lib/embeddings/provider_health.rb@@ -0,0 +1,85 @@+# frozen_string_literal: true++module DiscourseAi+ module Embeddings+ class ProviderPausedError < StandardError+ def initialize+ super("Embedding provider is temporarily paused")+ end+ end++ class ProviderHealth+ PAUSE_BACKOFFS = [30.seconds, 3.minutes, 20.minutes, 1.hour, 6.hours].freeze+ BACKOFF_STATE_TTL = 1.day++ class << self+ def paused?(definition)+ definition&.persisted? && Discourse.redis.exists?(pause_key(definition.id))+ end++ def request!(definition)+ state = provider_state(definition)+ raise ProviderPausedError if state&.first++ result = yield+ Discourse.redis.del(backoff_key(definition.id)) if state&.last+ result+ rescue DiscourseAi::Inference::EmbeddingInferenceError => error+ pause!(definition, error) if error.terminal?+ raise+ end++ def clear!(definition_or_id)+ id = definition_or_id.respond_to?(:id) ? definition_or_id.id : definition_or_id+ Discourse.redis.del(pause_key(id), backoff_key(id)) if id.present?+ end++ private++ def provider_state(definition)+ return if !definition&.persisted?++ Discourse.redis.mget(pause_key(definition.id), backoff_key(definition.id))+ end++ def pause!(definition, error)+ return if !definition&.persisted?++ level = [+ Discourse.redis.get(backoff_key(definition.id)).to_i,+ PAUSE_BACKOFFS.length - 1,+ ].min+ ttl = PAUSE_BACKOFFS[level]+ paused =+ Discourse.redis.set(+ pause_key(definition.id),+ error.category.to_s,+ nx: true,+ ex: ttl.to_i,+ )+ return if !paused++ Discourse.redis.set(+ backoff_key(definition.id),+ [level + 1, PAUSE_BACKOFFS.length - 1].min,+ ex: BACKOFF_STATE_TTL.to_i,+ )+ Rails.logger.warn(+ "Discourse AI embedding provider paused definition_id=#{definition.id} " \+ "provider=#{definition.provider} category=#{error.category} " \+ "duration=#{ttl.to_i}s status=#{error.http_status.inspect} " \+ "code=#{error.provider_error_code.inspect}",+ )+ end++ def pause_key(id)+ "discourse_ai:embedding_provider_paused:#{id}"+ end++ def backoff_key(id)+ "discourse_ai:embedding_provider_backoff:#{id}"+ end+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/embeddings/semantic_related.rb b/plugins/discourse-ai/lib/embeddings/semantic_related.rbindex 5ffcdbb59dc..806df3602be 100644--- a/plugins/discourse-ai/lib/embeddings/semantic_related.rb+++ b/plugins/discourse-ai/lib/embeddings/semantic_related.rb@@ -36,6 +36,9 @@ module DiscourseAiendendrescue DiscourseAi::Embeddings::Schema::MissingEmbeddingError+ definition = EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_selected_model)+ return [] if DiscourseAi::Embeddings::ProviderHealth.paused?(definition)+# avoid a flood of jobs when visiting topicif Discourse.redis.set(build_semantic_suggested_key(topic.id),diff --git a/plugins/discourse-ai/lib/embeddings/vector.rb b/plugins/discourse-ai/lib/embeddings/vector.rbindex 6b31ec533ff..d21eb8f7d37 100644--- a/plugins/discourse-ai/lib/embeddings/vector.rb+++ b/plugins/discourse-ai/lib/embeddings/vector.rb@@ -28,8 +28,6 @@ module DiscourseAischema = DiscourseAi::Embeddings::Schema.for(relation.first.class, vector_def: @vdef)- embedding_gen = vdef.inference_client-queued = 0results = Queue.new# map so we release the DB connection@@ -42,7 +40,7 @@ module DiscourseAipool.post doresults << { target: record, text: prepared_text, digest: new_digest }.merge(- embedding: embedding_gen.perform!(prepared_text),+ embedding: request_embedding!(prepared_text),)rescue StandardError => eresults << e@@ -61,11 +59,12 @@ module DiscourseAiqueued -= 1end- if errors.any?+ unexpected_errors = errors.reject { |error| expected_provider_error?(error) }+ if unexpected_errors.any?Discourse.warn_exception(- errors[0],+ unexpected_errors.first,message:- "Discourse AI: Errors during bulk classification: Failed to generate embeddings on #{errors.count} posts",+ "Discourse AI: Unexpected errors during bulk embedding generation: #{unexpected_errors.count}",)endensure@@ -82,7 +81,7 @@ module DiscourseAinew_digest = OpenSSL::Digest::SHA1.hexdigest(text)return if schema.find_by_target(target)&.digest == new_digest- embeddings = vdef.inference_client.perform!(text)+ embeddings = request_embedding!(text)schema.store(target, embeddings, new_digest)end@@ -91,10 +90,21 @@ module DiscourseAiprepared_text = vdef.prepare_query_text(text, asymmetric: asymmetric)return if prepared_text.blank?- vdef.inference_client.perform!(prepared_text)+ request_embedding!(prepared_text)endattr_reader :vdef++ private++ def request_embedding!(text)+ ProviderHealth.request!(vdef) { vdef.inference_client.perform!(text) }+ end++ def expected_provider_error?(error)+ error.is_a?(ProviderPausedError) ||+ error.is_a?(DiscourseAi::Inference::EmbeddingInferenceError)+ endendendenddiff --git a/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rb b/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rbindex 5177aa23241..bc0dc32dcdd 100644--- a/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rb+++ b/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rb@@ -23,14 +23,13 @@ module DiscourseAiconn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }response = conn.post(endpoint, payload.to_json, headers)- case response.status- when 200+ if response.status == 200JSON.parse(response.body, symbolize_names: true).dig(:result, :data).firstelse- Rails.logger.warn(- "Cloudflare Workers AI Embeddings failed with status: #{response.status} body: #{response.body}",- )- raise Net::HTTPBadResponse.new(response.body.to_s)+ raise EmbeddingInferenceError.from_response(+ provider: EmbeddingDefinition::CLOUDFLARE,+ response: response,+ )endendenddiff --git a/plugins/discourse-ai/lib/inference/embedding_inference_error.rb b/plugins/discourse-ai/lib/inference/embedding_inference_error.rbnew file mode 100644index 00000000000..567b8565182--- /dev/null+++ b/plugins/discourse-ai/lib/inference/embedding_inference_error.rb@@ -0,0 +1,127 @@+# frozen_string_literal: true++module DiscourseAi+ module Inference+ class EmbeddingInferenceError < StandardError+ TERMINAL_CATEGORIES = %i[+ quota_exhausted+ authentication_failed+ authorization_failed+ invalid_configuration+ ].freeze+ RETRYABLE_CATEGORIES = %i[rate_limited provider_unavailable].freeze+ MAX_MESSAGE_LENGTH = 500++ attr_reader :provider,+ :http_status,+ :provider_error_type,+ :provider_error_code,+ :category,+ :retry_after++ def self.from_response(provider:, response:)+ body =+ begin+ parsed = JSON.parse(response.body.to_s)+ parsed.is_a?(Hash) ? parsed : {}+ rescue JSON::ParserError+ {}+ end+ attributes = provider_attributes(provider, body)+ new(+ provider: provider,+ http_status: response.status,+ provider_error_type: attributes[:type],+ provider_error_code: attributes[:code],+ category: classify(provider, response.status.to_i, attributes),+ retry_after: retry_after(response.headers["retry-after"]),+ message: attributes[:message],+ )+ end++ def initialize(+ provider:,+ category:,+ http_status: nil,+ provider_error_type: nil,+ provider_error_code: nil,+ retry_after: nil,+ message: nil+ )+ @provider = provider.to_s.first(100)+ @category = category.to_sym+ @http_status = http_status&.to_i+ @provider_error_type = provider_error_type.to_s.first(100).presence+ @provider_error_code = provider_error_code.to_s.first(100).presence+ @retry_after = retry_after+ super(sanitize(message).presence || "Embedding provider request failed (#{category})")+ end++ def terminal?+ TERMINAL_CATEGORIES.include?(category)+ end++ def retryable?+ RETRYABLE_CATEGORIES.include?(category)+ end++ class << self+ private++ def provider_attributes(provider, body)+ error =+ case provider.to_s+ when EmbeddingDefinition::CLOUDFLARE+ Array(body["errors"]).first+ else+ body["error"]+ end+ return { message: error } if !error.is_a?(Hash)++ { type: error["type"] || error["status"], code: error["code"], message: error["message"] }+ end++ def classify(provider, status, attributes)+ evidence =+ [attributes[:type], attributes[:code]].compact.map { |value| value.to_s.downcase }+ if provider.to_s == EmbeddingDefinition::OPEN_AI &&+ evidence.intersect?(%w[insufficient_quota credit_balance_exhausted])+ return :quota_exhausted+ end++ return :authentication_failed if status == 401+ return :authorization_failed if status == 403+ return :rate_limited if status == 429+ return :provider_unavailable if status == 408 || status >= 500+ if evidence.intersect?(+ %w[invalid_argument invalid_model model_not_found deployment_notfound],+ )+ return :invalid_configuration+ end++ :request_rejected+ end++ def retry_after(value)+ return if value.blank?++ seconds = Integer(value, exception: false)+ seconds&.clamp(0, 300)+ end+ end++ private++ def sanitize(message)+ message+ .to_s+ .encode("UTF-8", invalid: :replace, undef: :replace, replace: "�")+ .gsub(/[[:cntrl:]]+/, " ")+ .gsub(/\b(?:sk|key)-[A-Za-z0-9_-]{8,}\b/i, "[REDACTED]")+ .gsub(/(?:bearer|api[-_ ]?key)\s*[:=]?\s*[^\s,;]+/i, "[REDACTED]")+ .squish+ .first(MAX_MESSAGE_LENGTH)+ end+ end+ end+enddiff --git a/plugins/discourse-ai/lib/inference/gemini_embeddings.rb b/plugins/discourse-ai/lib/inference/gemini_embeddings.rbindex a275dbdaa11..5db84ab9311 100644--- a/plugins/discourse-ai/lib/inference/gemini_embeddings.rb+++ b/plugins/discourse-ai/lib/inference/gemini_embeddings.rb@@ -23,14 +23,13 @@ module DiscourseAiconn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }response = conn.post(url, body.to_json, headers)- case response.status- when 200+ if response.status == 200JSON.parse(response.body, symbolize_names: true).dig(:embedding, :values)else- Rails.logger.warn(- "Google Gemini Embeddings failed with status: #{response.status} body: #{response.body}",- )- raise Net::HTTPBadResponse.new(response.body.to_s)+ raise EmbeddingInferenceError.from_response(+ provider: EmbeddingDefinition::GOOGLE,+ response: response,+ )endendenddiff --git a/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rb b/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rbindex 66690ae7b86..335e627e72c 100644--- a/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rb+++ b/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rb@@ -54,7 +54,13 @@ module DiscourseAienddef perform!(content)- response = do_request!(content)+ response = request(content)+ if response.status != 200+ raise EmbeddingInferenceError.from_response(+ provider: EmbeddingDefinition::HUGGING_FACE,+ response: response,+ )+ endJSON.parse(response.body, symbolize_names: true).firstend@@ -62,6 +68,13 @@ module DiscourseAiprivatedef do_request!(content)+ response = request(content)+ raise Net::HTTPBadResponse.new(response.body.to_s) if response.status != 200++ response+ end++ def request(content)headers = { "Referer" => referer, "Content-Type" => "application/json" }body = { inputs: content, truncate: true }.to_json@@ -71,11 +84,7 @@ module DiscourseAiendconn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }- response = conn.post(endpoint, body, headers)-- raise Net::HTTPBadResponse.new(response.body.to_s) if ![200].include?(response.status)-- response+ conn.post(endpoint, body, headers)endendenddiff --git a/plugins/discourse-ai/lib/inference/open_ai_embeddings.rb b/plugins/discourse-ai/lib/inference/open_ai_embeddings.rbindex 8a0e4bb642c..cbf83badad8 100644--- a/plugins/discourse-ai/lib/inference/open_ai_embeddings.rb+++ b/plugins/discourse-ai/lib/inference/open_ai_embeddings.rb@@ -27,14 +27,13 @@ module DiscourseAiconn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }response = conn.post(endpoint, payload.to_json, headers)- case response.status- when 200+ if response.status == 200JSON.parse(response.body, symbolize_names: true).dig(:data, 0, :embedding)else- Rails.logger.warn(- "OpenAI Embeddings failed with status: #{response.status} body: #{response.body}",- )- raise Net::HTTPBadResponse.new(response.body.to_s)+ raise EmbeddingInferenceError.from_response(+ provider: EmbeddingDefinition::OPEN_AI,+ response: response,+ )endendenddiff --git a/plugins/discourse-ai/lib/utils/search.rb b/plugins/discourse-ai/lib/utils/search.rbindex 72249b7fdea..e930db11e23 100644--- a/plugins/discourse-ai/lib/utils/search.rb+++ b/plugins/discourse-ai/lib/utils/search.rb@@ -87,8 +87,11 @@ module DiscourseAisemantic_results = nilbeginsemantic_results = semantic_search.search_for_topics(search.term, hyde: hyde)- rescue => e- Discourse.warn_exception(e, message: "Semantic search failed")+ rescue DiscourseAi::Inference::EmbeddingInferenceError,+ DiscourseAi::Embeddings::ProviderPausedError+ semantic_results = nil+ rescue StandardError => error+ Discourse.warn_exception(error, message: "Semantic search failed")endif semantic_resultsdiff --git a/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rbindex 8db75fae192..70d735472a3 100644--- a/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rb+++ b/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rb@@ -38,6 +38,27 @@ RSpec.describe Jobs::EmbeddingsBackfill do)end+ after do+ [vector_def, vector_def2].each do |definition|+ DiscourseAi::Embeddings::ProviderHealth.clear!(definition)+ end+ end++ it "skips a paused definition before generating embeddings" do+ error =+ DiscourseAi::Inference::EmbeddingInferenceError.new(+ provider: vector_def.provider,+ category: :quota_exhausted,+ )+ expect do+ DiscourseAi::Embeddings::ProviderHealth.request!(vector_def) { raise error }+ end.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)++ described_class.new.execute({})++ expect(a_request(:post, vector_def.url)).not_to have_been_made+ end+it "backfills topics based on bumped_at date" doJobs::EmbeddingsBackfill.new.execute({})diff --git a/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rb b/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rbindex d07a5d4edd4..79caeef4dda 100644--- a/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rb+++ b/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rb@@ -45,12 +45,15 @@ RSpec.describe DiscourseAi::Inference::CloudflareWorkersAi dolet(:response_status) { 500 }let(:response_body) { "Internal Server Error" }- it "raises a Net::HTTPBadResponse error" do+ it "raises a structured provider error without logging the response" doallow(Rails.logger).to receive(:warn)- expect { cloudflare_workers_ai.perform!(content) }.to raise_error(Net::HTTPBadResponse)- expect(Rails.logger).to have_received(:warn).with(- "Cloudflare Workers AI Embeddings failed with status: #{response_status} body: #{response_body}",- )++ expect { cloudflare_workers_ai.perform!(content) }.to raise_error(+ DiscourseAi::Inference::EmbeddingInferenceError,+ ) { |error|+ expect([error.http_status, error.category]).to eq([500, :provider_unavailable])+ }+ expect(Rails.logger).not_to have_received(:warn)endendenddiff --git a/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rb b/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rbindex 44058ef4c9a..1541d4e2846 100644--- a/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rb+++ b/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rb@@ -41,12 +41,15 @@ RSpec.describe DiscourseAi::Inference::GeminiEmbeddings dolet(:response_status) { 500 }let(:response_body) { "Internal Server Error" }- it "raises a Net::HTTPBadResponse error" do+ it "raises a structured provider error without logging the response" doallow(Rails.logger).to receive(:warn)- expect { gemini_embeddings.perform!(content) }.to raise_error(Net::HTTPBadResponse)- expect(Rails.logger).to have_received(:warn).with(- "Google Gemini Embeddings failed with status: #{response_status} body: #{response_body}",- )++ expect { gemini_embeddings.perform!(content) }.to raise_error(+ DiscourseAi::Inference::EmbeddingInferenceError,+ ) { |error|+ expect([error.http_status, error.category]).to eq([500, :provider_unavailable])+ }+ expect(Rails.logger).not_to have_received(:warn)endendenddiff --git a/plugins/discourse-ai/spec/lib/inference/hugging_face_text_embeddings_spec.rb b/plugins/discourse-ai/spec/lib/inference/hugging_face_text_embeddings_spec.rbnew file mode 100644index 00000000000..391b137f006--- /dev/null+++ b/plugins/discourse-ai/spec/lib/inference/hugging_face_text_embeddings_spec.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Inference::HuggingFaceTextEmbeddings do+ let(:endpoint) { "https://embeddings.example.com" }+ let(:client) { described_class.new(endpoint, "secret") }++ before do+ stub_request(:post, endpoint).to_return(status: 401, body: { error: "invalid key" }.to_json)+ end++ it "changes only the embedding error contract" do+ expect { client.perform!("content") }.to raise_error(+ DiscourseAi::Inference::EmbeddingInferenceError,+ )+ expect { client.classify_by_sentiment!("content") }.to raise_error(Net::HTTPBadResponse)+ end+enddiff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rbindex c748cb3cb5c..660fddf4b01 100644--- a/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rb@@ -10,6 +10,8 @@ describe DiscourseAi::Embeddings::EntryPoint doSiteSetting.ai_embeddings_selected_model = embedding_definition.idend+ after { DiscourseAi::Embeddings::ProviderHealth.clear!(embedding_definition) }+describe "registering event callbacks" docontext "when creating a topic" dolet(:creator) do@@ -32,6 +34,20 @@ describe DiscourseAi::Embeddings::EntryPoint doexpect { creator.create }.to change(Jobs::GenerateEmbeddings.jobs, :size).by(1) # topic_created AND post_createdend+ it "does not queue work while the selected provider is paused" do+ SiteSetting.ai_embeddings_enabled = true+ error =+ DiscourseAi::Inference::EmbeddingInferenceError.new(+ provider: embedding_definition.provider,+ category: :authentication_failed,+ )+ expect do+ DiscourseAi::Embeddings::ProviderHealth.request!(embedding_definition) { raise error }+ end.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)++ expect { creator.create }.not_to change(Jobs::GenerateEmbeddings.jobs, :size)+ end+it "does nothing if embeddings analysis is disabled" doSiteSetting.ai_embeddings_enabled = falsediff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rbindex c8da9a0d480..32250d6e8c0 100644--- a/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rb+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rb@@ -13,6 +13,8 @@ RSpec.describe Jobs::GenerateEmbeddings doSiteSetting.ai_embeddings_enabled = trueend+ after { DiscourseAi::Embeddings::ProviderHealth.clear!(vector_def) }+fab!(:topic)fab!(:post) { Fabricate(:post, post_number: 1, topic: topic) }@@ -41,5 +43,19 @@ RSpec.describe Jobs::GenerateEmbeddings doexpect(posts_schema.find_by_embedding(expected_embedding).post_id).to eq(post.id)end++ it "quietly skips jobs after a terminal provider failure" do+ error =+ DiscourseAi::Inference::EmbeddingInferenceError.new(+ provider: vector_def.provider,+ category: :authentication_failed,+ )+ expect do+ DiscourseAi::Embeddings::ProviderHealth.request!(vector_def) { raise error }+ end.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)++ expect { job.execute(target_id: topic.id, target_type: "Topic") }.not_to raise_error+ expect(a_request(:post, vector_def.url)).not_to have_been_made+ endendenddiff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rbnew file mode 100644index 00000000000..c852756f2d8--- /dev/null+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb@@ -0,0 +1,102 @@+# frozen_string_literal: true++RSpec.describe DiscourseAi::Embeddings::ProviderHealth do+ fab!(:definition, :open_ai_embedding_def)++ after { described_class.clear!(definition) }++ def error(category)+ DiscourseAi::Inference::EmbeddingInferenceError.new(+ provider: definition.provider,+ category: category,+ http_status: 429,+ provider_error_code: category,+ )+ end++ def pause_key+ "discourse_ai:embedding_provider_paused:#{definition.id}"+ end++ def pause_ttl+ Discourse.redis.ttl(pause_key)+ end++ it "pauses once for terminal failures and skips later provider calls" do+ allow(Rails.logger).to receive(:warn)++ expect {+ described_class.request!(definition) { raise error(:quota_exhausted) }+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)++ expect {+ described_class.request!(definition) { raise error(:quota_exhausted) }+ }.to raise_error(DiscourseAi::Embeddings::ProviderPausedError)++ contacted = false+ expect { described_class.request!(definition) { contacted = true } }.to raise_error(+ DiscourseAi::Embeddings::ProviderPausedError,+ )+ expect(contacted).to eq(false)+ expect(Rails.logger).to have_received(:warn).once+ end++ it "progressively increases pauses for consecutive terminal failures" do+ expected_backoffs = [30.seconds, 3.minutes, 20.minutes, 1.hour, 6.hours, 6.hours]++ expected_backoffs.each do |expected_backoff|+ expect {+ described_class.request!(definition) { raise error(:authentication_failed) }+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)+ expect(pause_ttl).to be_between(expected_backoff.to_i - 1, expected_backoff.to_i)++ Discourse.redis.del(pause_key)+ end+ end++ it "resets the backoff after a successful provider request" do+ expect {+ described_class.request!(definition) { raise error(:quota_exhausted) }+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)+ Discourse.redis.del(pause_key)++ expect(described_class.request!(definition) { :embedding }).to eq(:embedding)+ expect {+ described_class.request!(definition) { raise error(:quota_exhausted) }+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)++ expect(pause_ttl).to be_between(29, 30)+ end++ it "does not pause temporary rate limits" do+ expect { described_class.request!(definition) { raise error(:rate_limited) } }.to raise_error(+ DiscourseAi::Inference::EmbeddingInferenceError,+ )++ expect(described_class).not_to be_paused(definition)+ end++ it "does not share health state with unsaved definitions" do+ unsaved =+ EmbeddingDefinition.new(definition.attributes.except("id", "created_at", "updated_at"))++ expect { described_class.request!(unsaved) { raise error(:quota_exhausted) } }.to raise_error(+ DiscourseAi::Inference::EmbeddingInferenceError,+ )+ expect(described_class).not_to be_paused(unsaved)+ end++ it "clears the pause and backoff after the definition changes" do+ expect {+ described_class.request!(definition) { raise error(:authentication_failed) }+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)++ definition.clear_provider_pause+ expect(described_class).not_to be_paused(definition)++ expect {+ described_class.request!(definition) { raise error(:authentication_failed) }+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)+ expect(pause_ttl).to be_between(29, 30)+ end+enddiff --git a/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rbindex 7809b8fc20d..87cbfc7b108 100644--- a/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rb+++ b/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rb@@ -276,7 +276,7 @@ RSpec.describe DiscourseAi::Admin::AiEmbeddingsController doexpect(response).to be_successfulexpect(response.parsed_body["success"]).to eq(false)- expect(response.parsed_body["error"]).to eq(error_message.to_json)+ expect(response.parsed_body["error"]).to eq(error_message[:error])endendenddiff --git a/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rb b/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rbindex 01135ba98d8..46096e206cf 100644--- a/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rb+++ b/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rb@@ -60,4 +60,28 @@ describe DiscourseAi::Inference::OpenAiEmbeddings doexpect(result).to eq([0.0, 0.1])end++ it "distinguishes OpenAI quota exhaustion from temporary rate limiting" do+ url = "https://api.openai.com/v1/embeddings"+ client = described_class.new(url, api_key, model, dimensions)++ [+ ["rate_limit_exceeded", :rate_limited, true],+ ["insufficient_quota", :quota_exhausted, false],+ ["credit_balance_exhausted", :quota_exhausted, false],+ ].each do |code, category, retryable|+ stub_request(:post, url).to_return(+ status: 429,+ body: { error: { type: code, code: code, message: "request failed" } }.to_json,+ )++ expect { client.perform!("hello") }.to raise_error(+ DiscourseAi::Inference::EmbeddingInferenceError,+ ) do |error|+ expect([error.provider_error_code, error.category, error.retryable?]).to eq(+ [code, category, retryable],+ )+ end+ end+ endend