Skip to content

A platform for community discussion. Free, open, simple.

FIX: Pause failing embedding providers (#42539)

Samfeb3fba6a3bd
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb
index 8c0951f4369..fbde5e7f9bb 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/admin/ai_embeddings_controller.rb
@@ -113,8 +113,9 @@ module DiscourseAi
DiscourseAi::Embeddings::Vector.new(embedding_def).vector_from("this is a test")
render json: { success: true }
- rescue Net::HTTPBadResponse => e
- render json: { success: false, error: e.message }
+ rescue DiscourseAi::Inference::EmbeddingInferenceError,
+ DiscourseAi::Embeddings::ProviderPausedError => error
+ render json: { success: false, error: error.message }
end
private
diff --git a/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rb b/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rb
index 1f8665ce5b5..bb837d6f866 100644
--- a/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rb
+++ b/plugins/discourse-ai/app/controllers/discourse_ai/embeddings/embeddings_controller.rb
@@ -71,8 +71,8 @@ module DiscourseAi
.each { |topic_post| grouped_results.add(topic_post) }
rescue Discourse::InvalidAccess
return render_json_error(I18n.t("invalid_access"), status: 403)
- rescue Net::HTTPBadResponse => e
- Rails.logger.warn("Semantic search embedding generation failed: #{e.message}")
+ rescue DiscourseAi::Inference::EmbeddingInferenceError,
+ DiscourseAi::Embeddings::ProviderPausedError
end
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
@@ -110,8 +110,8 @@ module DiscourseAi
semantic_search
.search_for_topics(query, _page = 1, hyde: false)
.each { |topic_post| grouped_results.add(topic_post) }
- rescue Net::HTTPBadResponse => e
- Rails.logger.warn("Quick search embedding generation failed: #{e.message}")
+ rescue DiscourseAi::Inference::EmbeddingInferenceError,
+ DiscourseAi::Embeddings::ProviderPausedError
end
render_serialized(grouped_results, GroupedSearchResultSerializer, result: grouped_results)
diff --git a/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rb b/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rb
index ab3ea167813..5413b4d72d3 100644
--- a/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rb
+++ b/plugins/discourse-ai/app/jobs/regular/generate_embeddings.rb
@@ -16,7 +16,14 @@ module Jobs
return if topic.private_message? && !SiteSetting.ai_embeddings_generate_for_pms
return if post.raw.blank?
- DiscourseAi::Embeddings::Vector.instance.generate_representation_from(target)
+ definition = EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_selected_model)
+ return if DiscourseAi::Embeddings::ProviderHealth.paused?(definition)
+
+ DiscourseAi::Embeddings::Vector.new(definition).generate_representation_from(target)
+ rescue DiscourseAi::Inference::EmbeddingInferenceError => error
+ raise if !error.terminal?
+ rescue DiscourseAi::Embeddings::ProviderPausedError
+ nil
end
end
end
diff --git a/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rb b/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rb
index 163534cc240..329119aa7b6 100644
--- a/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rb
+++ b/plugins/discourse-ai/app/jobs/scheduled/embeddings_backfill.rb
@@ -33,6 +33,8 @@ module Jobs
topic_work_list << backfill_vector if backfill_vector
topic_work_list.each do |vector|
+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)
+
rebaked = 0
table_name = DiscourseAi::Embeddings::Schema::TOPICS_TABLE
vector_def = vector.vdef
@@ -48,6 +50,7 @@ module Jobs
rebaked += populate_topic_embeddings(vector, topics.limit(limit - rebaked))
+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)
next if rebaked >= limit
# Then, we'll try to backfill embeddings for topics that have outdated
@@ -60,6 +63,7 @@ module Jobs
rebaked += populate_topic_embeddings(vector, relation, force: true)
+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)
next if rebaked >= limit
# Finally, we'll try to backfill embeddings for topics that have outdated
@@ -72,6 +76,7 @@ module Jobs
populate_topic_embeddings(vector, relation, force: true)
+ next if DiscourseAi::Embeddings::ProviderHealth.paused?(vector.vdef)
next unless SiteSetting.ai_embeddings_per_post_enabled
# Now for posts
diff --git a/plugins/discourse-ai/app/models/embedding_definition.rb b/plugins/discourse-ai/app/models/embedding_definition.rb
index 03c2df8a5ca..19bacd4ec7a 100644
--- a/plugins/discourse-ai/app/models/embedding_definition.rb
+++ b/plugins/discourse-ai/app/models/embedding_definition.rb
@@ -131,11 +131,17 @@ class EmbeddingDefinition < ActiveRecord::Base
validate :api_key_or_secret_present
after_create :create_indexes
+ after_update_commit :clear_provider_pause
+ after_destroy_commit :clear_provider_pause
def create_indexes
Jobs.enqueue(:manage_embedding_def_search_index, id: id)
end
+ def clear_provider_pause
+ DiscourseAi::Embeddings::ProviderHealth.clear!(id)
+ end
+
def tokenizer
tokenizer_class.constantize
end
diff --git a/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rb b/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rb
index 84b1c821e84..b245e41ef7d 100644
--- a/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rb
+++ b/plugins/discourse-ai/lib/configuration/embedding_defs_validator.rb
@@ -25,7 +25,8 @@ module DiscourseAi
end
true
- rescue Net::HTTPBadResponse
+ rescue DiscourseAi::Inference::EmbeddingInferenceError,
+ DiscourseAi::Embeddings::ProviderPausedError
false
end
diff --git a/plugins/discourse-ai/lib/embeddings/entry_point.rb b/plugins/discourse-ai/lib/embeddings/entry_point.rb
index 1c625ee1577..61b839cf166 100644
--- a/plugins/discourse-ai/lib/embeddings/entry_point.rb
+++ b/plugins/discourse-ai/lib/embeddings/entry_point.rb
@@ -53,7 +53,9 @@ module DiscourseAi
# embeddings generation.
callback =
Proc.new do |target|
+ definition = EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_selected_model)
if DiscourseAi::Embeddings.enabled? &&
+ !DiscourseAi::Embeddings::ProviderHealth.paused?(definition) &&
(target.is_a?(Topic) || SiteSetting.ai_embeddings_per_post_enabled)
Jobs.enqueue(
:generate_embeddings,
@@ -77,10 +79,15 @@ module DiscourseAi
if DiscourseAi::Embeddings.enabled?
query = [args[:title], args[:raw]].join("\n\n")
- DiscourseAi::Embeddings::SemanticSearch
- .new(args[:guardian])
- .similar_topic_ids_to(query, candidates: args[:candidates])
- .each { |similar_topic_id| plugin_candidate_ids << similar_topic_id }
+ begin
+ DiscourseAi::Embeddings::SemanticSearch
+ .new(args[:guardian])
+ .similar_topic_ids_to(query, candidates: args[:candidates])
+ .each { |similar_topic_id| plugin_candidate_ids << similar_topic_id }
+ rescue DiscourseAi::Inference::EmbeddingInferenceError,
+ DiscourseAi::Embeddings::ProviderPausedError
+ # Ordinary similar-topic candidates remain available.
+ end
end
end
end
diff --git a/plugins/discourse-ai/lib/embeddings/provider_health.rb b/plugins/discourse-ai/lib/embeddings/provider_health.rb
new file mode 100644
index 00000000000..19a3856108f
--- /dev/null
+++ b/plugins/discourse-ai/lib/embeddings/provider_health.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Embeddings
+ class ProviderPausedError < StandardError
+ def initialize
+ super("Embedding provider is temporarily paused")
+ end
+ end
+
+ class ProviderHealth
+ PAUSE_BACKOFFS = [30.seconds, 3.minutes, 20.minutes, 1.hour, 6.hours].freeze
+ BACKOFF_STATE_TTL = 1.day
+
+ class << self
+ def paused?(definition)
+ definition&.persisted? && Discourse.redis.exists?(pause_key(definition.id))
+ end
+
+ def request!(definition)
+ state = provider_state(definition)
+ raise ProviderPausedError if state&.first
+
+ result = yield
+ Discourse.redis.del(backoff_key(definition.id)) if state&.last
+ result
+ rescue DiscourseAi::Inference::EmbeddingInferenceError => error
+ pause!(definition, error) if error.terminal?
+ raise
+ end
+
+ def clear!(definition_or_id)
+ id = definition_or_id.respond_to?(:id) ? definition_or_id.id : definition_or_id
+ Discourse.redis.del(pause_key(id), backoff_key(id)) if id.present?
+ end
+
+ private
+
+ def provider_state(definition)
+ return if !definition&.persisted?
+
+ Discourse.redis.mget(pause_key(definition.id), backoff_key(definition.id))
+ end
+
+ def pause!(definition, error)
+ return if !definition&.persisted?
+
+ level = [
+ Discourse.redis.get(backoff_key(definition.id)).to_i,
+ PAUSE_BACKOFFS.length - 1,
+ ].min
+ ttl = PAUSE_BACKOFFS[level]
+ paused =
+ Discourse.redis.set(
+ pause_key(definition.id),
+ error.category.to_s,
+ nx: true,
+ ex: ttl.to_i,
+ )
+ return if !paused
+
+ Discourse.redis.set(
+ backoff_key(definition.id),
+ [level + 1, PAUSE_BACKOFFS.length - 1].min,
+ ex: BACKOFF_STATE_TTL.to_i,
+ )
+ Rails.logger.warn(
+ "Discourse AI embedding provider paused definition_id=#{definition.id} " \
+ "provider=#{definition.provider} category=#{error.category} " \
+ "duration=#{ttl.to_i}s status=#{error.http_status.inspect} " \
+ "code=#{error.provider_error_code.inspect}",
+ )
+ end
+
+ def pause_key(id)
+ "discourse_ai:embedding_provider_paused:#{id}"
+ end
+
+ def backoff_key(id)
+ "discourse_ai:embedding_provider_backoff:#{id}"
+ end
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/embeddings/semantic_related.rb b/plugins/discourse-ai/lib/embeddings/semantic_related.rb
index 5ffcdbb59dc..806df3602be 100644
--- a/plugins/discourse-ai/lib/embeddings/semantic_related.rb
+++ b/plugins/discourse-ai/lib/embeddings/semantic_related.rb
@@ -36,6 +36,9 @@ module DiscourseAi
end
end
rescue DiscourseAi::Embeddings::Schema::MissingEmbeddingError
+ definition = EmbeddingDefinition.find_by(id: SiteSetting.ai_embeddings_selected_model)
+ return [] if DiscourseAi::Embeddings::ProviderHealth.paused?(definition)
+
# avoid a flood of jobs when visiting topic
if Discourse.redis.set(
build_semantic_suggested_key(topic.id),
diff --git a/plugins/discourse-ai/lib/embeddings/vector.rb b/plugins/discourse-ai/lib/embeddings/vector.rb
index 6b31ec533ff..d21eb8f7d37 100644
--- a/plugins/discourse-ai/lib/embeddings/vector.rb
+++ b/plugins/discourse-ai/lib/embeddings/vector.rb
@@ -28,8 +28,6 @@ module DiscourseAi
schema = DiscourseAi::Embeddings::Schema.for(relation.first.class, vector_def: @vdef)
- embedding_gen = vdef.inference_client
-
queued = 0
results = Queue.new
# map so we release the DB connection
@@ -42,7 +40,7 @@ module DiscourseAi
pool.post do
results << { target: record, text: prepared_text, digest: new_digest }.merge(
- embedding: embedding_gen.perform!(prepared_text),
+ embedding: request_embedding!(prepared_text),
)
rescue StandardError => e
results << e
@@ -61,11 +59,12 @@ module DiscourseAi
queued -= 1
end
- if errors.any?
+ unexpected_errors = errors.reject { |error| expected_provider_error?(error) }
+ if unexpected_errors.any?
Discourse.warn_exception(
- errors[0],
+ unexpected_errors.first,
message:
- "Discourse AI: Errors during bulk classification: Failed to generate embeddings on #{errors.count} posts",
+ "Discourse AI: Unexpected errors during bulk embedding generation: #{unexpected_errors.count}",
)
end
ensure
@@ -82,7 +81,7 @@ module DiscourseAi
new_digest = OpenSSL::Digest::SHA1.hexdigest(text)
return if schema.find_by_target(target)&.digest == new_digest
- embeddings = vdef.inference_client.perform!(text)
+ embeddings = request_embedding!(text)
schema.store(target, embeddings, new_digest)
end
@@ -91,10 +90,21 @@ module DiscourseAi
prepared_text = vdef.prepare_query_text(text, asymmetric: asymmetric)
return if prepared_text.blank?
- vdef.inference_client.perform!(prepared_text)
+ request_embedding!(prepared_text)
end
attr_reader :vdef
+
+ private
+
+ def request_embedding!(text)
+ ProviderHealth.request!(vdef) { vdef.inference_client.perform!(text) }
+ end
+
+ def expected_provider_error?(error)
+ error.is_a?(ProviderPausedError) ||
+ error.is_a?(DiscourseAi::Inference::EmbeddingInferenceError)
+ end
end
end
end
diff --git a/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rb b/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rb
index 5177aa23241..bc0dc32dcdd 100644
--- a/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rb
+++ b/plugins/discourse-ai/lib/inference/cloudflare_workers_ai.rb
@@ -23,14 +23,13 @@ module DiscourseAi
conn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }
response = conn.post(endpoint, payload.to_json, headers)
- case response.status
- when 200
+ if response.status == 200
JSON.parse(response.body, symbolize_names: true).dig(:result, :data).first
else
- Rails.logger.warn(
- "Cloudflare Workers AI Embeddings failed with status: #{response.status} body: #{response.body}",
- )
- raise Net::HTTPBadResponse.new(response.body.to_s)
+ raise EmbeddingInferenceError.from_response(
+ provider: EmbeddingDefinition::CLOUDFLARE,
+ response: response,
+ )
end
end
end
diff --git a/plugins/discourse-ai/lib/inference/embedding_inference_error.rb b/plugins/discourse-ai/lib/inference/embedding_inference_error.rb
new file mode 100644
index 00000000000..567b8565182
--- /dev/null
+++ b/plugins/discourse-ai/lib/inference/embedding_inference_error.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module DiscourseAi
+ module Inference
+ class EmbeddingInferenceError < StandardError
+ TERMINAL_CATEGORIES = %i[
+ quota_exhausted
+ authentication_failed
+ authorization_failed
+ invalid_configuration
+ ].freeze
+ RETRYABLE_CATEGORIES = %i[rate_limited provider_unavailable].freeze
+ MAX_MESSAGE_LENGTH = 500
+
+ attr_reader :provider,
+ :http_status,
+ :provider_error_type,
+ :provider_error_code,
+ :category,
+ :retry_after
+
+ def self.from_response(provider:, response:)
+ body =
+ begin
+ parsed = JSON.parse(response.body.to_s)
+ parsed.is_a?(Hash) ? parsed : {}
+ rescue JSON::ParserError
+ {}
+ end
+ attributes = provider_attributes(provider, body)
+ new(
+ provider: provider,
+ http_status: response.status,
+ provider_error_type: attributes[:type],
+ provider_error_code: attributes[:code],
+ category: classify(provider, response.status.to_i, attributes),
+ retry_after: retry_after(response.headers["retry-after"]),
+ message: attributes[:message],
+ )
+ end
+
+ def initialize(
+ provider:,
+ category:,
+ http_status: nil,
+ provider_error_type: nil,
+ provider_error_code: nil,
+ retry_after: nil,
+ message: nil
+ )
+ @provider = provider.to_s.first(100)
+ @category = category.to_sym
+ @http_status = http_status&.to_i
+ @provider_error_type = provider_error_type.to_s.first(100).presence
+ @provider_error_code = provider_error_code.to_s.first(100).presence
+ @retry_after = retry_after
+ super(sanitize(message).presence || "Embedding provider request failed (#{category})")
+ end
+
+ def terminal?
+ TERMINAL_CATEGORIES.include?(category)
+ end
+
+ def retryable?
+ RETRYABLE_CATEGORIES.include?(category)
+ end
+
+ class << self
+ private
+
+ def provider_attributes(provider, body)
+ error =
+ case provider.to_s
+ when EmbeddingDefinition::CLOUDFLARE
+ Array(body["errors"]).first
+ else
+ body["error"]
+ end
+ return { message: error } if !error.is_a?(Hash)
+
+ { type: error["type"] || error["status"], code: error["code"], message: error["message"] }
+ end
+
+ def classify(provider, status, attributes)
+ evidence =
+ [attributes[:type], attributes[:code]].compact.map { |value| value.to_s.downcase }
+ if provider.to_s == EmbeddingDefinition::OPEN_AI &&
+ evidence.intersect?(%w[insufficient_quota credit_balance_exhausted])
+ return :quota_exhausted
+ end
+
+ return :authentication_failed if status == 401
+ return :authorization_failed if status == 403
+ return :rate_limited if status == 429
+ return :provider_unavailable if status == 408 || status >= 500
+ if evidence.intersect?(
+ %w[invalid_argument invalid_model model_not_found deployment_notfound],
+ )
+ return :invalid_configuration
+ end
+
+ :request_rejected
+ end
+
+ def retry_after(value)
+ return if value.blank?
+
+ seconds = Integer(value, exception: false)
+ seconds&.clamp(0, 300)
+ end
+ end
+
+ private
+
+ def sanitize(message)
+ message
+ .to_s
+ .encode("UTF-8", invalid: :replace, undef: :replace, replace: "�")
+ .gsub(/[[:cntrl:]]+/, " ")
+ .gsub(/\b(?:sk|key)-[A-Za-z0-9_-]{8,}\b/i, "[REDACTED]")
+ .gsub(/(?:bearer|api[-_ ]?key)\s*[:=]?\s*[^\s,;]+/i, "[REDACTED]")
+ .squish
+ .first(MAX_MESSAGE_LENGTH)
+ end
+ end
+ end
+end
diff --git a/plugins/discourse-ai/lib/inference/gemini_embeddings.rb b/plugins/discourse-ai/lib/inference/gemini_embeddings.rb
index a275dbdaa11..5db84ab9311 100644
--- a/plugins/discourse-ai/lib/inference/gemini_embeddings.rb
+++ b/plugins/discourse-ai/lib/inference/gemini_embeddings.rb
@@ -23,14 +23,13 @@ module DiscourseAi
conn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }
response = conn.post(url, body.to_json, headers)
- case response.status
- when 200
+ if response.status == 200
JSON.parse(response.body, symbolize_names: true).dig(:embedding, :values)
else
- Rails.logger.warn(
- "Google Gemini Embeddings failed with status: #{response.status} body: #{response.body}",
- )
- raise Net::HTTPBadResponse.new(response.body.to_s)
+ raise EmbeddingInferenceError.from_response(
+ provider: EmbeddingDefinition::GOOGLE,
+ response: response,
+ )
end
end
end
diff --git a/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rb b/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rb
index 66690ae7b86..335e627e72c 100644
--- a/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rb
+++ b/plugins/discourse-ai/lib/inference/hugging_face_text_embeddings.rb
@@ -54,7 +54,13 @@ module DiscourseAi
end
def perform!(content)
- response = do_request!(content)
+ response = request(content)
+ if response.status != 200
+ raise EmbeddingInferenceError.from_response(
+ provider: EmbeddingDefinition::HUGGING_FACE,
+ response: response,
+ )
+ end
JSON.parse(response.body, symbolize_names: true).first
end
@@ -62,6 +68,13 @@ module DiscourseAi
private
def do_request!(content)
+ response = request(content)
+ raise Net::HTTPBadResponse.new(response.body.to_s) if response.status != 200
+
+ response
+ end
+
+ def request(content)
headers = { "Referer" => referer, "Content-Type" => "application/json" }
body = { inputs: content, truncate: true }.to_json
@@ -71,11 +84,7 @@ module DiscourseAi
end
conn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }
- response = conn.post(endpoint, body, headers)
-
- raise Net::HTTPBadResponse.new(response.body.to_s) if ![200].include?(response.status)
-
- response
+ conn.post(endpoint, body, headers)
end
end
end
diff --git a/plugins/discourse-ai/lib/inference/open_ai_embeddings.rb b/plugins/discourse-ai/lib/inference/open_ai_embeddings.rb
index 8a0e4bb642c..cbf83badad8 100644
--- a/plugins/discourse-ai/lib/inference/open_ai_embeddings.rb
+++ b/plugins/discourse-ai/lib/inference/open_ai_embeddings.rb
@@ -27,14 +27,13 @@ module DiscourseAi
conn = Faraday.new { |f| f.adapter FinalDestination::FaradayAdapter }
response = conn.post(endpoint, payload.to_json, headers)
- case response.status
- when 200
+ if response.status == 200
JSON.parse(response.body, symbolize_names: true).dig(:data, 0, :embedding)
else
- Rails.logger.warn(
- "OpenAI Embeddings failed with status: #{response.status} body: #{response.body}",
- )
- raise Net::HTTPBadResponse.new(response.body.to_s)
+ raise EmbeddingInferenceError.from_response(
+ provider: EmbeddingDefinition::OPEN_AI,
+ response: response,
+ )
end
end
end
diff --git a/plugins/discourse-ai/lib/utils/search.rb b/plugins/discourse-ai/lib/utils/search.rb
index 72249b7fdea..e930db11e23 100644
--- a/plugins/discourse-ai/lib/utils/search.rb
+++ b/plugins/discourse-ai/lib/utils/search.rb
@@ -87,8 +87,11 @@ module DiscourseAi
semantic_results = nil
begin
semantic_results = semantic_search.search_for_topics(search.term, hyde: hyde)
- rescue => e
- Discourse.warn_exception(e, message: "Semantic search failed")
+ rescue DiscourseAi::Inference::EmbeddingInferenceError,
+ DiscourseAi::Embeddings::ProviderPausedError
+ semantic_results = nil
+ rescue StandardError => error
+ Discourse.warn_exception(error, message: "Semantic search failed")
end
if semantic_results
diff --git a/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rb b/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rb
index 8db75fae192..70d735472a3 100644
--- a/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rb
+++ b/plugins/discourse-ai/spec/jobs/scheduled/embeddings_backfill_spec.rb
@@ -38,6 +38,27 @@ RSpec.describe Jobs::EmbeddingsBackfill do
)
end
+ after do
+ [vector_def, vector_def2].each do |definition|
+ DiscourseAi::Embeddings::ProviderHealth.clear!(definition)
+ end
+ end
+
+ it "skips a paused definition before generating embeddings" do
+ error =
+ DiscourseAi::Inference::EmbeddingInferenceError.new(
+ provider: vector_def.provider,
+ category: :quota_exhausted,
+ )
+ expect do
+ DiscourseAi::Embeddings::ProviderHealth.request!(vector_def) { raise error }
+ end.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+
+ described_class.new.execute({})
+
+ expect(a_request(:post, vector_def.url)).not_to have_been_made
+ end
+
it "backfills topics based on bumped_at date" do
Jobs::EmbeddingsBackfill.new.execute({})
diff --git a/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rb b/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rb
index d07a5d4edd4..79caeef4dda 100644
--- a/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rb
+++ b/plugins/discourse-ai/spec/lib/inference/cloudflare_workers_ai_spec.rb
@@ -45,12 +45,15 @@ RSpec.describe DiscourseAi::Inference::CloudflareWorkersAi do
let(:response_status) { 500 }
let(:response_body) { "Internal Server Error" }
- it "raises a Net::HTTPBadResponse error" do
+ it "raises a structured provider error without logging the response" do
allow(Rails.logger).to receive(:warn)
- expect { cloudflare_workers_ai.perform!(content) }.to raise_error(Net::HTTPBadResponse)
- expect(Rails.logger).to have_received(:warn).with(
- "Cloudflare Workers AI Embeddings failed with status: #{response_status} body: #{response_body}",
- )
+
+ expect { cloudflare_workers_ai.perform!(content) }.to raise_error(
+ DiscourseAi::Inference::EmbeddingInferenceError,
+ ) { |error|
+ expect([error.http_status, error.category]).to eq([500, :provider_unavailable])
+ }
+ expect(Rails.logger).not_to have_received(:warn)
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rb b/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rb
index 44058ef4c9a..1541d4e2846 100644
--- a/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rb
+++ b/plugins/discourse-ai/spec/lib/inference/gemini_embeddings_spec.rb
@@ -41,12 +41,15 @@ RSpec.describe DiscourseAi::Inference::GeminiEmbeddings do
let(:response_status) { 500 }
let(:response_body) { "Internal Server Error" }
- it "raises a Net::HTTPBadResponse error" do
+ it "raises a structured provider error without logging the response" do
allow(Rails.logger).to receive(:warn)
- expect { gemini_embeddings.perform!(content) }.to raise_error(Net::HTTPBadResponse)
- expect(Rails.logger).to have_received(:warn).with(
- "Google Gemini Embeddings failed with status: #{response_status} body: #{response_body}",
- )
+
+ expect { gemini_embeddings.perform!(content) }.to raise_error(
+ DiscourseAi::Inference::EmbeddingInferenceError,
+ ) { |error|
+ expect([error.http_status, error.category]).to eq([500, :provider_unavailable])
+ }
+ expect(Rails.logger).not_to have_received(:warn)
end
end
end
diff --git a/plugins/discourse-ai/spec/lib/inference/hugging_face_text_embeddings_spec.rb b/plugins/discourse-ai/spec/lib/inference/hugging_face_text_embeddings_spec.rb
new file mode 100644
index 00000000000..391b137f006
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/inference/hugging_face_text_embeddings_spec.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Inference::HuggingFaceTextEmbeddings do
+ let(:endpoint) { "https://embeddings.example.com" }
+ let(:client) { described_class.new(endpoint, "secret") }
+
+ before do
+ stub_request(:post, endpoint).to_return(status: 401, body: { error: "invalid key" }.to_json)
+ end
+
+ it "changes only the embedding error contract" do
+ expect { client.perform!("content") }.to raise_error(
+ DiscourseAi::Inference::EmbeddingInferenceError,
+ )
+ expect { client.classify_by_sentiment!("content") }.to raise_error(Net::HTTPBadResponse)
+ end
+end
diff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rb
index c748cb3cb5c..660fddf4b01 100644
--- a/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/entry_point_spec.rb
@@ -10,6 +10,8 @@ describe DiscourseAi::Embeddings::EntryPoint do
SiteSetting.ai_embeddings_selected_model = embedding_definition.id
end
+ after { DiscourseAi::Embeddings::ProviderHealth.clear!(embedding_definition) }
+
describe "registering event callbacks" do
context "when creating a topic" do
let(:creator) do
@@ -32,6 +34,20 @@ describe DiscourseAi::Embeddings::EntryPoint do
expect { creator.create }.to change(Jobs::GenerateEmbeddings.jobs, :size).by(1) # topic_created AND post_created
end
+ it "does not queue work while the selected provider is paused" do
+ SiteSetting.ai_embeddings_enabled = true
+ error =
+ DiscourseAi::Inference::EmbeddingInferenceError.new(
+ provider: embedding_definition.provider,
+ category: :authentication_failed,
+ )
+ expect do
+ DiscourseAi::Embeddings::ProviderHealth.request!(embedding_definition) { raise error }
+ end.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+
+ expect { creator.create }.not_to change(Jobs::GenerateEmbeddings.jobs, :size)
+ end
+
it "does nothing if embeddings analysis is disabled" do
SiteSetting.ai_embeddings_enabled = false
diff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rb
index c8da9a0d480..32250d6e8c0 100644
--- a/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rb
+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/jobs/generate_embeddings_spec.rb
@@ -13,6 +13,8 @@ RSpec.describe Jobs::GenerateEmbeddings do
SiteSetting.ai_embeddings_enabled = true
end
+ after { DiscourseAi::Embeddings::ProviderHealth.clear!(vector_def) }
+
fab!(:topic)
fab!(:post) { Fabricate(:post, post_number: 1, topic: topic) }
@@ -41,5 +43,19 @@ RSpec.describe Jobs::GenerateEmbeddings do
expect(posts_schema.find_by_embedding(expected_embedding).post_id).to eq(post.id)
end
+
+ it "quietly skips jobs after a terminal provider failure" do
+ error =
+ DiscourseAi::Inference::EmbeddingInferenceError.new(
+ provider: vector_def.provider,
+ category: :authentication_failed,
+ )
+ expect do
+ DiscourseAi::Embeddings::ProviderHealth.request!(vector_def) { raise error }
+ end.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+
+ expect { job.execute(target_id: topic.id, target_type: "Topic") }.not_to raise_error
+ expect(a_request(:post, vector_def.url)).not_to have_been_made
+ end
end
end
diff --git a/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb
new file mode 100644
index 00000000000..c852756f2d8
--- /dev/null
+++ b/plugins/discourse-ai/spec/lib/modules/embeddings/provider_health_spec.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+RSpec.describe DiscourseAi::Embeddings::ProviderHealth do
+ fab!(:definition, :open_ai_embedding_def)
+
+ after { described_class.clear!(definition) }
+
+ def error(category)
+ DiscourseAi::Inference::EmbeddingInferenceError.new(
+ provider: definition.provider,
+ category: category,
+ http_status: 429,
+ provider_error_code: category,
+ )
+ end
+
+ def pause_key
+ "discourse_ai:embedding_provider_paused:#{definition.id}"
+ end
+
+ def pause_ttl
+ Discourse.redis.ttl(pause_key)
+ end
+
+ it "pauses once for terminal failures and skips later provider calls" do
+ allow(Rails.logger).to receive(:warn)
+
+ expect {
+ described_class.request!(definition) { raise error(:quota_exhausted) }
+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+
+ expect {
+ described_class.request!(definition) { raise error(:quota_exhausted) }
+ }.to raise_error(DiscourseAi::Embeddings::ProviderPausedError)
+
+ contacted = false
+ expect { described_class.request!(definition) { contacted = true } }.to raise_error(
+ DiscourseAi::Embeddings::ProviderPausedError,
+ )
+ expect(contacted).to eq(false)
+ expect(Rails.logger).to have_received(:warn).once
+ end
+
+ it "progressively increases pauses for consecutive terminal failures" do
+ expected_backoffs = [30.seconds, 3.minutes, 20.minutes, 1.hour, 6.hours, 6.hours]
+
+ expected_backoffs.each do |expected_backoff|
+ expect {
+ described_class.request!(definition) { raise error(:authentication_failed) }
+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+ expect(pause_ttl).to be_between(expected_backoff.to_i - 1, expected_backoff.to_i)
+
+ Discourse.redis.del(pause_key)
+ end
+ end
+
+ it "resets the backoff after a successful provider request" do
+ expect {
+ described_class.request!(definition) { raise error(:quota_exhausted) }
+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+ Discourse.redis.del(pause_key)
+
+ expect(described_class.request!(definition) { :embedding }).to eq(:embedding)
+ expect {
+ described_class.request!(definition) { raise error(:quota_exhausted) }
+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+
+ expect(pause_ttl).to be_between(29, 30)
+ end
+
+ it "does not pause temporary rate limits" do
+ expect { described_class.request!(definition) { raise error(:rate_limited) } }.to raise_error(
+ DiscourseAi::Inference::EmbeddingInferenceError,
+ )
+
+ expect(described_class).not_to be_paused(definition)
+ end
+
+ it "does not share health state with unsaved definitions" do
+ unsaved =
+ EmbeddingDefinition.new(definition.attributes.except("id", "created_at", "updated_at"))
+
+ expect { described_class.request!(unsaved) { raise error(:quota_exhausted) } }.to raise_error(
+ DiscourseAi::Inference::EmbeddingInferenceError,
+ )
+ expect(described_class).not_to be_paused(unsaved)
+ end
+
+ it "clears the pause and backoff after the definition changes" do
+ expect {
+ described_class.request!(definition) { raise error(:authentication_failed) }
+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+
+ definition.clear_provider_pause
+ expect(described_class).not_to be_paused(definition)
+
+ expect {
+ described_class.request!(definition) { raise error(:authentication_failed) }
+ }.to raise_error(DiscourseAi::Inference::EmbeddingInferenceError)
+ expect(pause_ttl).to be_between(29, 30)
+ end
+end
diff --git a/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rb b/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rb
index 7809b8fc20d..87cbfc7b108 100644
--- a/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rb
+++ b/plugins/discourse-ai/spec/requests/admin/ai_embeddings_controller_spec.rb
@@ -276,7 +276,7 @@ RSpec.describe DiscourseAi::Admin::AiEmbeddingsController do
expect(response).to be_successful
expect(response.parsed_body["success"]).to eq(false)
- expect(response.parsed_body["error"]).to eq(error_message.to_json)
+ expect(response.parsed_body["error"]).to eq(error_message[:error])
end
end
end
diff --git a/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rb b/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rb
index 01135ba98d8..46096e206cf 100644
--- a/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rb
+++ b/plugins/discourse-ai/spec/shared/inference/openai_embeddings_spec.rb
@@ -60,4 +60,28 @@ describe DiscourseAi::Inference::OpenAiEmbeddings do
expect(result).to eq([0.0, 0.1])
end
+
+ it "distinguishes OpenAI quota exhaustion from temporary rate limiting" do
+ url = "https://api.openai.com/v1/embeddings"
+ client = described_class.new(url, api_key, model, dimensions)
+
+ [
+ ["rate_limit_exceeded", :rate_limited, true],
+ ["insufficient_quota", :quota_exhausted, false],
+ ["credit_balance_exhausted", :quota_exhausted, false],
+ ].each do |code, category, retryable|
+ stub_request(:post, url).to_return(
+ status: 429,
+ body: { error: { type: code, code: code, message: "request failed" } }.to_json,
+ )
+
+ expect { client.perform!("hello") }.to raise_error(
+ DiscourseAi::Inference::EmbeddingInferenceError,
+ ) do |error|
+ expect([error.provider_error_code, error.category, error.retryable?]).to eq(
+ [code, category, retryable],
+ )
+ end
+ end
+ end
end