Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add embedder circuit breaker and recovery (#78296)

Chris Truter020b02932d83
diff --git a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
index 607c81d3daf..bb6cf1a547d 100644
--- a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
+++ b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj
@@ -31,6 +31,14 @@
(set! *warn-on-reflection* true)
+(defn- search-embedding-model
+ "Translate the neutral embedding descriptor's dimension key to semantic search's internal key."
+ [embedding-model]
+ (cond-> embedding-model
+ (and (nil? (:vector-dimensions embedding-model))
+ (some? (:model-dimensions embedding-model)))
+ (assoc :vector-dimensions (:model-dimensions embedding-model))))
+
(defn get-embeddings-batch
"Return a sequential collection of embedding vectors, in the same order as the input texts.
@@ -41,4 +49,4 @@
`opts` — optional keyword opts (e.g. `:type :doc`). Accepts alternating kwargs or a single trailing map;
forwarded as kwargs into the multimethod, which destructures with `& {:as opts}`."
[embedding-model texts & {:as opts}]
- (apply semantic-search/get-embeddings-batch embedding-model texts (mapcat identity opts)))
+ (apply semantic-search/get-embeddings-batch (search-embedding-model embedding-model) texts (mapcat identity opts)))
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
index c3e06f03c80..fbf5524aeab 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj
@@ -2,6 +2,8 @@
(:require
[clj-http.client :as http]
[clojure.string :as str]
+ [diehard.circuit-breaker :as dh.cb]
+ [flatland.ordered.set :refer [ordered-set]]
[metabase-enterprise.semantic-search.models.token-tracking :as semantic.models.token-tracking]
[metabase-enterprise.semantic-search.settings :as semantic-settings]
[metabase.analytics-interface.core :as analytics]
@@ -16,6 +18,7 @@
(:import
[com.knuddels.jtokkit Encodings]
[com.knuddels.jtokkit.api Encoding EncodingType]
+ [dev.failsafe CircuitBreaker]
[java.net ConnectException]))
(set! *warn-on-reflection* true)
@@ -107,7 +110,7 @@
(do
(log/warn
(format "Skipping text that exceeds maximum measure per batch: %s"
- (str (subs text 0 10) "..."))
+ (str (subs text 0 (min 10 (count text))) "..."))
{:measure text-measure :threshold threshold})
acc)
@@ -136,54 +139,267 @@
(defn- dispatch-provider [embedding-model & _] (:provider embedding-model))
(defmulti get-embedding
- "Returns a single embedding vector for the given text"
+ "Returns a single embedding vector for the given text.
+ `opts` (kwargs, honoured by the production providers; ollama ignores them):
+ - `:record-tokens?` — write a token-tracking row for the call
+ - `:type` — what the embedding is for (`:query`/`:index`), recorded with the tokens
+ - `:snowplow?` — ai-service only, default true; synthetic callers (e.g. the health probe) pass
+ false so the call emits no token_usage event"
{:arglists '([embedding-model text & opts])} dispatch-provider)
(defmulti get-embeddings-batch
- "Returns a sequential collection of embedding vectors, in the same order as the input texts."
+ "Returns a sequential collection of embedding vectors, in the same order as the input texts.
+ Takes the same `opts` as [[get-embedding]], minus `:snowplow?` (batch callers are all organic)."
{:arglists '([embedding-model texts & opts])} dispatch-provider)
+(defmulti embedder-circuit-endpoint
+ "Return the remote endpoint that identifies `embedding-model`'s circuit, or nil when its provider does not
+ use the remote-service circuit breaker."
+ {:arglists '([embedding-model])} dispatch-provider)
+
+(defmethod embedder-circuit-endpoint :default [_] nil)
+
(defmulti pull-model
"If a model needs to be downloaded (which is the case for ollama), downloads it."
{:arglists '([embedding-model])} dispatch-provider)
+;;;; Embedding-service circuit breaker
+;;;
+;;; The embedding service is a remote HTTP dependency; when it's down, every semantic-search / NLQ query
+;;; would otherwise re-attempt (and re-time-out) against it. A circuit breaker fails fast after repeated
+;;; failures, and its state doubles as a health signal: an open breaker means real traffic is currently
+;;; failing. State transitions surface the affected health checks immediately (see the listeners) rather
+;;; than waiting for the daily job. Thresholds are fixed (the breaker is built once); the setting is a
+;;; runtime kill switch checked per call.
+
+(def ^:private embedder-circuit-breaker-failure-threshold
+ "Consecutive embedding-service failures that trip the breaker open." 5)
+
+(def ^:private embedder-circuit-breaker-success-threshold
+ "Consecutive successes in half-open that close the breaker again." 2)
+
+(def ^:private embedder-circuit-breaker-delay-ms
+ "How long the breaker stays open before it allows a half-open trial call." 30000)
+
+(def ^:private embedding-http-timeouts
+ "clj-http timeout opts merged into every embedding-service request. Without an explicit :socket-timeout the
+ client waits forever, so a blackholed service (accepts connections, never responds) would hang callers
+ indefinitely -- and the breaker would never open, since only completed failures count toward it."
+ {:connection-timeout 10000
+ :socket-timeout 60000})
+
+(defonce ^{:doc "Insertion-ordered set of `(fn [state])` hooks run on every breaker state change.
+ Health namespaces `conj` a hook here (inverting the dep -- they require this module) to re-persist their
+ embedder-dependent check on a transition, so an outage or recovery surfaces in minutes, not the next daily report.
+ Ordered-set: `conj` is reload-idempotent and load-ordered, so the semantic hook clears the probe cache
+ before the NLQ hook reads it. Register a var so a REPL redef takes effect live."}
+ embedder-circuit-state-change-hooks
+ (atom (ordered-set)))
+
+(defonce ^:private embedder-circuit-hook-runner
+ ;; One agent, so transitions run strictly in arrival order: concurrent futures would let a slow earlier
+ ;; transition (e.g. its check blocked on a probe against a down service) persist a stale result AFTER a
+ ;; later recovery transition's healthy one, leaving a pre-recovery "unreachable" row standing.
+ ;; :continue so an unexpected action error can't wedge the agent for the process lifetime.
+ (agent nil
+ :error-mode :continue
+ :error-handler (fn [_agent e] (log/error e "Embedder circuit hook runner errored"))))
+
+(defn- on-embedder-circuit-state-change!
+ "Run the registered [[embedder-circuit-state-change-hooks]] off the failsafe callback thread -- one
+ transition at a time in arrival order, hooks in registration order, each isolated so one failing hook
+ doesn't starve the rest."
+ [state]
+ ;; Log level tracks the resulting state's severity:
+ ;; :open -> WARN (degradation)
+ ;; :half-open -> INFO (probing)
+ ;; :closed -> INFO (recovered)
+ ;; Recovery is still captured level-independently -- the hooks below persist a health row per transition.
+ (if (= :open state)
+ (log/warn "Embedding service circuit breaker opened" {:state state})
+ (log/info "Embedding service circuit breaker changed state" {:state state}))
+ (send-off
+ embedder-circuit-hook-runner
+ (fn [_]
+ (doseq [hook @embedder-circuit-state-change-hooks]
+ (try
+ (hook state)
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/error e "Embedder circuit state-change hook failed")))))))
+
+(defn- new-embedder-circuit-breaker []
+ (dh.cb/circuit-breaker
+ {:failure-threshold embedder-circuit-breaker-failure-threshold
+ :success-threshold embedder-circuit-breaker-success-threshold
+ :delay-ms embedder-circuit-breaker-delay-ms
+ :on-open (fn [_] (on-embedder-circuit-state-change! :open))
+ :on-half-open (fn [_] (on-embedder-circuit-state-change! :half-open))
+ :on-close (fn [_] (on-embedder-circuit-state-change! :closed))}))
+
+(defonce ^:private embedder-circuit-breakers
+ ;; Provider settings can differ between semantic search and data complexity. Keying by endpoint prevents
+ ;; one unavailable service from fast-failing calls to an unrelated, healthy service.
+ (atom {}))
+
+(declare get-configured-model)
+
+(defn- circuit-breaker-for
+ ^CircuitBreaker [endpoint]
+ (or (get @embedder-circuit-breakers endpoint)
+ (get (swap! embedder-circuit-breakers
+ #(if (contains? % endpoint)
+ %
+ (assoc % endpoint (new-embedder-circuit-breaker))))
+ endpoint)))
+
+(def ^:dynamic *bypass-circuit-breaker*
+ "Bind true to run an embedding call without consulting or tripping the breaker.
+ The health probe binds it so the probe stays an independent signal and can't flip the breaker from inside
+ a state-change listener (which would recurse)."
+ false)
+
+(defn embedder-circuit-state
+ "Return the configured embedder circuit's state (`:closed`, `:open`, or `:half-open`), or nil when the
+ provider has no circuit endpoint. The optional `endpoint` arity inspects another endpoint's isolated circuit."
+ ([]
+ (embedder-circuit-state (embedder-circuit-endpoint (get-configured-model))))
+ ([endpoint]
+ (when endpoint
+ (dh.cb/state (circuit-breaker-for endpoint)))))
+
+(defn embedder-circuit-untrusted?
+ "Whether the breaker is enabled and not closed -- i.e. open or half-open, so it doesn't yet trust the
+ embedding service (open short-circuits calls; half-open is on a single trial).
+ False when the breaker is disabled or the provider has no circuit endpoint."
+ []
+ (boolean
+ (and (semantic-settings/semantic-search-embedder-circuit-breaker-enabled)
+ (when-let [state (embedder-circuit-state)]
+ (not= :closed state)))))
+
+(def ^:private request-specific-statuses
+ "HTTP statuses caused by a particular input, rather than the embedding service as a whole."
+ #{400 404 413 422})
+
+(defn- service-failure?
+ [e]
+ (let [data (ex-data e)
+ cause (some-> e ex-cause ex-data)
+ status (or (:status data) (:status cause))
+ reason (or (:cause data) (:cause cause))]
+ (and (not (contains? request-specific-statuses status))
+ (not= :embedder/unexpected-dimensions reason))))
+
+(defn- circuit-open-ex
+ [endpoint]
+ (ex-info "embedding service unavailable (circuit open)"
+ (cond-> {:status 502, :cause :embedder/circuit-open}
+ endpoint (assoc :endpoint endpoint))))
+
+(defn- call-through-embedder-breaker
+ "Run `thunk` under the embedder circuit breaker, unless it is bypassed (probe) or disabled (kill switch).
+ Circuits are isolated by endpoint. Service-wide failures update the circuit; request- and model-specific
+ failures propagate without changing its history. An open circuit throws a 502 `ex-info` with
+ `:cause :embedder/circuit-open`."
+ [thunk & {:keys [endpoint]}]
+ (if (or (nil? endpoint)
+ *bypass-circuit-breaker*
+ (not (semantic-settings/semantic-search-embedder-circuit-breaker-enabled)))
+ (thunk)
+ (let [^CircuitBreaker breaker (circuit-breaker-for endpoint)]
+ (when-not (dh.cb/allow-execution? breaker)
+ (throw (circuit-open-ex endpoint)))
+ (let [execution-state (dh.cb/state breaker)]
+ (try
+ (let [result (thunk)]
+ (.recordSuccess breaker)
+ result)
+ (catch InterruptedException e
+ (when (= :half-open execution-state)
+ (.recordFailure breaker))
+ (throw e))
+ (catch Exception e
+ (cond
+ (service-failure? e) (.recordException breaker e)
+ (= :half-open execution-state) (.recordSuccess breaker))
+ (throw e))
+ (catch Throwable t
+ ;; Every half-open permit must record an outcome, even when the JVM-level failure itself should
+ ;; propagate unchanged rather than contribute to the service-failure history.
+ (when (= :half-open execution-state)
+ (.recordFailure breaker))
+ (throw t)))))))
+
+(defn- validate-embeddings!
+ [embeddings expected-count expected-dimensions]
+ (when-not (= expected-count (count embeddings))
+ (throw (ex-info "Embedding response contained an unexpected number of vectors"
+ {:cause :embedder/invalid-response
+ :expected-count expected-count
+ :actual-count (count embeddings)})))
+ (doseq [embedding embeddings]
+ (when-not (= expected-dimensions (count embedding))
+ (throw (ex-info "Embedding response contained a vector with unexpected dimensions"
+ {:cause :embedder/unexpected-dimensions
+ :expected-dimensions expected-dimensions
+ :actual-dimensions (count embedding)})))
+ (when-not (every? #(Double/isFinite (double %)) embedding)
+ (throw (ex-info "Embedding response contained a non-finite value"
+ {:cause :embedder/invalid-response}))))
+ embeddings)
+
;;;; Ollama impl
-(defn- ollama-get-embedding [model-name text]
+(def ^:private ollama-embeddings-endpoint
+ "http://localhost:11434/api/embeddings") ;; TODO: we should make the host configurable
+
+(defmethod embedder-circuit-endpoint "ollama" [_] ollama-embeddings-endpoint)
+
+(defn- ollama-get-embedding [model-name vector-dimensions text]
(try
;; TODO count ollama tokens into :metabase-search/semantic-embedding-tokens?
(log/debug "Generating Ollama embedding for text of length:" (count text))
- (-> (http/post "http://localhost:11434/api/embeddings" ;; TODO: we should make the host configurable
- {:headers {"Content-Type" "application/json"}
- :body (json/encode {:model model-name
- :prompt text})})
- :body
- (json/decode true)
- :embedding)
+ (let [embedding (-> (http/post ollama-embeddings-endpoint
+ (merge embedding-http-timeouts
+ {:headers {"Content-Type" "application/json"}
+ :body (json/encode {:model model-name
+ :prompt text})}))
+ :body
+ (json/decode true)
+ :embedding)]
+ (first (validate-embeddings! [embedding] 1 vector-dimensions)))
(catch Exception e
(log/error e "Failed to generate Ollama embedding for text of length:" (count text))
(throw e))))
-(defn- ollama-get-embeddings-batch [model-name texts]
- ;; Ollama doesn't have a native batch API, so we fall back to individual calls
- ;; No special batching needed for Ollama - just process all texts
- (log/debug "Generating" (count texts) "Ollama embeddings (using individual calls)")
- (mapv #(ollama-get-embedding model-name %) texts))
-
(defn- ollama-pull-model [model-name]
(try
(log/debug "Pulling embedding model from Ollama...")
(http/post "http://localhost:11434/api/pull" ;; TODO: make the host configurable
- {:headers {"Content-Type" "application/json"}
- :body (json/encode {:model model-name})})
+ (merge embedding-http-timeouts
+ {:headers {"Content-Type" "application/json"}
+ :body (json/encode {:model model-name})}))
(catch Exception e
(log/error e "Failed to pull embedding model")
(throw e))))
;; Ollama is not used in production. Token tracking is not implemented.
-(defmethod get-embedding "ollama" [{:keys [model-name]} text & {:as _opts}] (ollama-get-embedding model-name text))
-(defmethod get-embeddings-batch "ollama" [{:keys [model-name]} texts & {:as _opts}] (ollama-get-embeddings-batch model-name texts))
-(defmethod pull-model "ollama" [{:keys [model-name]}] (ollama-pull-model model-name))
+(defmethod get-embedding "ollama" [{:keys [model-name vector-dimensions]} text & {:as _opts}]
+ (call-through-embedder-breaker #(ollama-get-embedding model-name vector-dimensions text)
+ :endpoint ollama-embeddings-endpoint))
+
+(defmethod get-embeddings-batch "ollama" [{:keys [model-name vector-dimensions]} texts & {:as _opts}]
+ ;; Ollama doesn't have a native batch API, so we fall back to individual calls. Each call goes through the
+ ;; breaker on its own, so an outage mid-batch fast-fails the remaining texts instead of timing out on each.
+ (log/debug "Generating" (count texts) "Ollama embeddings (using individual calls)")
+ (mapv (fn [text]
+ (call-through-embedder-breaker #(ollama-get-embedding model-name vector-dimensions text)
+ :endpoint ollama-embeddings-endpoint))
+ texts))
+
+(defmethod pull-model "ollama" [{:keys [model-name]}] (ollama-pull-model model-name))
;;;; OpenAI-compatible embedding service impl (shared by "ai-service" and "openai" providers)
@@ -196,25 +412,28 @@
(str/starts-with? model-name "text-embedding-3"))))
(mu/defn- openai-compatible-get-embeddings-batch
- "Call an OpenAI-compatible /v1/embeddings endpoint. Shared implementation for both
- the `ai-service` and `openai` providers.
+ "Call an OpenAI-compatible /v1/embeddings endpoint. The breaker guards only the remote request and
+ response decoding; analytics and token persistence happen after it returns.
`:provider` — label for analytics (e.g. \"ai-service\", \"openai\")
`:endpoint` — full URL including /v1/embeddings
`:api-key` — Bearer token. If empty ai service proxying is assumed and premium-embedding-token is
used for authentication
`:model-name` — model identifier sent in the request body
+ `:vector-dimensions` — expected dimensions of each returned vector
`:texts` — collection of input strings
`:record-tokens?` — true writes a `semantic_search_token_tracking` row, false skips it.
`:snowplow?` — optional; when true fires a Snowplow `token_usage` event
`:extra-body` — optional; merged into the request body (e.g. `{:dimensions 1024}`)
`:type` — optional; forwarded to the token-tracking row"
- [{:keys [provider endpoint api-key model-name texts record-tokens? extra-body snowplow?] :as opts}
+ [{:keys [provider endpoint api-key model-name vector-dimensions texts record-tokens? extra-body snowplow?]
+ :as opts}
:- [:map
[:provider :string]
[:endpoint :string]
[:api-key {:optional true} [:maybe :string]]
[:model-name :string]
+ [:vector-dimensions pos-int?]
[:texts [:sequential :string]]
[:record-tokens? :boolean]
[:snowplow? {:optional true} [:maybe :boolean]]
@@ -222,24 +441,32 @@
(try
(log/debug (str "Calling " provider " embeddings API")
{:endpoint endpoint :documents (count texts) :tokens (count-tokens-batch texts)})
- (let [start-ms (u/start-timer)
- {:keys [usage data]} (-> (http/post endpoint
- {:headers
- (merge {"Content-Type" "application/json"}
- (if (and (empty? api-key)
- (= "ai-service" provider))
- {"x-metabase-instance-token"
- (u/prog1 (premium-features/premium-embedding-token)
- (when (nil? <>)
- (throw (ex-info "Premium embedding token not set"
- {:provider provider}))))}
- {"Authorization" (str "Bearer " api-key)}))
- :body (json/encode (merge {:model model-name
- :input texts
- :encoding_format "base64"}
- extra-body))})
- :body
- (json/decode true))
+ (let [headers (merge {"Content-Type" "application/json"}
+ (if (and (empty? api-key) (= "ai-service" provider))
+ {"x-metabase-instance-token"
+ (u/prog1 (premium-features/premium-embedding-token)
+ (when (nil? <>)
+ (throw (ex-info "Premium embedding token not set"
+ {:provider provider}))))}
+ {"Authorization" (str "Bearer " api-key)}))
+ request (merge embedding-http-timeouts
+ {:headers headers
+ :body (json/encode
+ (merge {:model model-name
+ :input texts
+ :encoding_format "base64"}
+ extra-body))})
+ start-ms (u/start-timer)
+ {:keys [usage embeddings]}
+ (call-through-embedder-breaker
+ #(let [{:keys [usage data]} (-> (http/post endpoint request)
+ :body
+ (json/decode true))]
+ {:usage usage
+ :embeddings (validate-embeddings! (decode-embeddings data)
+ (count texts)
+ vector-dimensions)})
+ :endpoint endpoint)
total-tokens (:total_tokens usage 0)
prompt-tokens (:prompt_tokens usage total-tokens)]
(analytics/inc! :metabase-search/semantic-embedding-tokens
@@ -259,15 +486,18 @@
:tag "embedding_generation"}))
(when record-tokens?
(semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens))
- (decode-embeddings data))
+ embeddings)
(catch ConnectException e
(log/error e (str "Failed to connect to " provider) {:endpoint endpoint})
(throw (ex-info (str provider " unavailable (connection refused)")
{:status 502 :endpoint endpoint}
e)))
(catch Exception e
- (log/error e (str provider " embeddings API call failed")
- {:documents (count texts) :tokens (count-tokens-batch texts)})
+ ;; The breaker transition already logs the outage once. Fast-failed calls while it remains open are
+ ;; expected and can be frequent, so do not emit a redundant error for every guarded request.
+ (when-not (= :embedder/circuit-open (:cause (ex-data e)))
+ (log/error e (str "Failed to generate " provider " embeddings")
+ {:documents (count texts) :tokens (count-tokens-batch texts)}))
(throw e))))
;;;; Embedding-service provider
@@ -296,27 +526,32 @@
{:settings ["ee-embedding-service-base-url"
"ai-service-base-url"]}))))
+(defmethod embedder-circuit-endpoint "ai-service" [_]
+ (first (embedding-service-resolve-config!)))
+
(defmethod get-embedding "ai-service"
- [{:keys [model-name]} text & {:keys [record-tokens? type]}]
+ [{:keys [model-name vector-dimensions]} text & {:keys [record-tokens? type snowplow?] :or {snowplow? true}}]
(let [[endpoint api-key] (embedding-service-resolve-config!)]
(first (openai-compatible-get-embeddings-batch
{:provider "ai-service"
:endpoint endpoint
:api-key api-key
:model-name model-name
+ :vector-dimensions vector-dimensions
:texts [text]
- :snowplow? true
+ :snowplow? snowplow?
:record-tokens? record-tokens?
:type type}))))
(defmethod get-embeddings-batch "ai-service"
- [{:keys [model-name]} texts & {:keys [record-tokens? type]}]
+ [{:keys [model-name vector-dimensions]} texts & {:keys [record-tokens? type]}]
(let [[endpoint api-key] (embedding-service-resolve-config!)]
(openai-compatible-get-embeddings-batch
{:provider "ai-service"
:endpoint endpoint
:api-key api-key
:model-name model-name
+ :vector-dimensions vector-dimensions
:texts texts
:snowplow? true
:record-tokens? record-tokens?
@@ -335,6 +570,9 @@
(throw (ex-info "OpenAI API key not configured" {:setting "llm-openai-api-key"})))
[(str (semantic-settings/openai-api-base-url) "/v1/embeddings") api-key]))
+(defmethod embedder-circuit-endpoint "openai" [_]
+ (first (openai-resolve-config!)))
+
(defmethod get-embedding "openai"
[embedding-model text & {:keys [record-tokens? type]}]
(let [[endpoint api-key] (openai-resolve-config!)]
@@ -343,6 +581,7 @@
:endpoint endpoint
:api-key api-key
:model-name (:model-name embedding-model)
+ :vector-dimensions (:vector-dimensions embedding-model)
:texts [text]
:record-tokens? record-tokens?
:extra-body (when (supports-dimensions? embedding-model)
@@ -357,6 +596,7 @@
:endpoint endpoint
:api-key api-key
:model-name (:model-name embedding-model)
+ :vector-dimensions (:vector-dimensions embedding-model)
:texts texts
:record-tokens? record-tokens?
:extra-body (when (supports-dimensions? embedding-model)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding_health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding_health.clj
new file mode 100644
index 00000000000..f189f0436bf
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding_health.clj
@@ -0,0 +1,123 @@
+(ns metabase-enterprise.semantic-search.embedding-health
+ "Embedding-service liveness and circuit recovery shared by semantic search and entity retrieval."
+ (:require
+ [clojure.core.memoize :as memoize]
+ [metabase-enterprise.semantic-search.embedding :as embedding]
+ [metabase.util.log :as log])
+ (:import
+ (java.util.concurrent Executors ScheduledExecutorService ThreadFactory TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(defn- probe-embedding-service
+ []
+ (try
+ ;; This liveness signal must not mutate the breaker; recovery trials are scheduled separately below.
+ (binding [embedding/*bypass-circuit-breaker* true]
+ (embedding/get-embedding (embedding/get-configured-model)
+ "health check"
+ {:type :query, :record-tokens? false, :snowplow? false}))
+ {:reachable? true, :error nil}
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/debug e "Embedding service health probe failed")
+ {:reachable? false, :error (ex-message e)})))
+
+(def ^:private embedding-service-reachable?*
+ (memoize/ttl probe-embedding-service :ttl/threshold (* 60 1000)))
+
+(defn embedding-service-reachable?
+ "Probe the configured embedding service. Results are shared for a 60-second window."
+ []
+ (embedding-service-reachable?*))
+
+(declare request-circuit-recovery!)
+
+(defn embedding-problem
+ "Return an embedding-service problem, or nil when the service and circuit are ready.
+ A reachable probe schedules circuit recovery when an idle workload has not supplied trial calls."
+ []
+ (let [{:keys [reachable? error]} (embedding-service-reachable?)]
+ (cond
+ (not reachable?)
+ (str "embedding service unreachable: " error)
+
+ (embedding/embedder-circuit-untrusted?)
+ (do
+ (request-circuit-recovery!)
+ "embedder circuit open (probe reachable; breaker still guarding calls)"))))
+
+(def ^:private recovery-delay-ms 30000)
+
+(def ^:private recovery-cooldown-ns
+ (* recovery-delay-ms 1000000))
+
+(defonce ^:private ^ScheduledExecutorService recovery-scheduler
+ (Executors/newSingleThreadScheduledExecutor
+ (reify ThreadFactory
+ (newThread [_ runnable]
+ (doto (Thread. runnable "embedding-circuit-recovery")
+ (.setDaemon true))))))
+
+(defonce ^:private recovery-state
+ (atom {:running? false, :last-start-ns nil}))
+
+(defn- claim-recovery! [now-ns]
+ (loop []
+ (let [{:keys [running? last-start-ns] :as state} @recovery-state]
+ (cond
+ running? false
+ (and last-start-ns (< (- now-ns last-start-ns) recovery-cooldown-ns)) false
+ (compare-and-set! recovery-state state {:running? true, :last-start-ns now-ns}) true
+ :else (recur)))))
+
+(defn- run-recovery-probe! []
+ (embedding/get-embedding (embedding/get-configured-model)
+ "health check"
+ {:type :query, :record-tokens? false, :snowplow? false}))
+
+(defn- recover-circuit! []
+ (try
+ ;; More than the current two-success threshold keeps this correct if that threshold is raised modestly.
+ (loop [remaining 4]
+ (when (and (pos? remaining) (embedding/embedder-circuit-untrusted?))
+ (run-recovery-probe!)
+ (recur (dec remaining))))
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/debug e "Embedding service circuit recovery probe failed"))
+ (finally
+ (swap! recovery-state assoc :running? false)))
+ ;; A still-untrusted breaker needs another delayed trial even when no user traffic or health consumer runs.
+ (when (embedding/embedder-circuit-untrusted?)
+ (request-circuit-recovery!)))
+
+(defn- schedule-recovery! [f]
+ (.schedule recovery-scheduler ^Runnable f (long recovery-delay-ms) TimeUnit/MILLISECONDS))
+
+(defn request-circuit-recovery!
+ "Schedule a throttled breaker-controlled probe for the configured semantic-search embedder after the breaker's
+ open delay. Semantic search and library retrieval share this embedder. Returns promptly."
+ []
+ (when (and (embedding/embedder-circuit-untrusted?)
+ (claim-recovery! (System/nanoTime)))
+ (try
+ (schedule-recovery! recover-circuit!)
+ (catch Exception e
+ (swap! recovery-state assoc :running? false)
+ (log/error e "Failed to schedule embedding service circuit recovery"))))
+ nil)
+
+(defn- clear-probe-cache-on-recovery! [state]
+ (when (= :closed state)
+ (memoize/memo-clear! embedding-service-reachable?*)))
+
+(defn- request-recovery-on-open! [state]
+ (when (= :open state)
+ (request-circuit-recovery!)))
+
+;; Register the shared cache hook before either consumer registers its health-row hook.
+(swap! embedding/embedder-circuit-state-change-hooks conj #'clear-probe-cache-on-recovery!)
+(swap! embedding/embedder-circuit-state-change-hooks conj #'request-recovery-on-open!)
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
index 330cec96293..07908499c6c 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj
@@ -1,5 +1,6 @@
(ns metabase-enterprise.semantic-search.init
(:require
+ [metabase-enterprise.semantic-search.embedding-health]
[metabase-enterprise.semantic-search.events]
[metabase-enterprise.semantic-search.settings]
[metabase-enterprise.semantic-search.task.index-cleanup]
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
index 34a90d8a90e..c4abc0798aa 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj
@@ -85,6 +85,17 @@
:export? false
:doc false)
+(defsetting semantic-search-embedder-circuit-breaker-enabled
+ (deferred-tru
+ (str "Wrap embedding-service calls in a circuit breaker that fails fast after repeated failures. "
+ "Runtime kill switch; the breaker thresholds are fixed."))
+ :type :boolean
+ :default true
+ :encryption :no
+ :export? false
+ :visibility :internal
+ :doc false)
+
(defsetting openai-max-tokens-per-batch
(deferred-tru "The maximum number of tokens sent in a single embedding API call.")
:type :integer
diff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
index c8c8037f824..494c0287436 100644
--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj
@@ -31,6 +31,10 @@
(def ^:private test-entity-ids (atom 0))
+(defmethod semantic-search/get-embeddings-batch ::dimension-translation-test
+ [embedding-model _texts & _opts]
+ [embedding-model])
+
(defn- entity
"Build a fake entity map for scoring tests. Uses a monotonically-increasing counter for `:id`
so every test entity is distinct (and so test failures print stable, readable ids)."
@@ -744,6 +748,13 @@
(embedder [{:id 1 :name "orders" :kind :table}])
(is (false? (:record-tokens? @captured)))))))
+(deftest ^:parallel embeddings-client-translates-neutral-dimension-key-test
+ (let [[translated]
+ (embeddings/get-embeddings-batch
+ {:provider ::dimension-translation-test, :model-name "fake", :model-dimensions 384}
+ ["orders"])]
+ (is (= 384 (:vector-dimensions translated)))))
+
(deftest ^:sequential provider-embedder-splits-names-before-calling-provider-test
(testing "provider-embedder splits names on _, -, ., and camelCase before sending to get-embeddings-batch"
(let [captured (atom nil)
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj
new file mode 100644
index 00000000000..9c09f1afc6e
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj
@@ -0,0 +1,333 @@
+(ns metabase-enterprise.semantic-search.embedding-circuit-breaker-test
+ (:require
+ [clj-http.client :as http]
+ [clojure.test :refer :all]
+ [diehard.circuit-breaker :as dh.cb]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]
+ [metabase-enterprise.semantic-search.models.token-tracking :as token-tracking]
+ [metabase-enterprise.semantic-search.settings :as semantic.settings]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.health-inspector.core :as health-inspector]
+ [metabase.llm.settings :as llm.settings]
+ [metabase.test :as mt])
+ (:import
+ (java.util.concurrent CountDownLatch TimeUnit)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private test-endpoint "https://circuit-test.example/v1/embeddings")
+(def ^:private test-provider ::circuit-test)
+(def ^:private test-embedding-model {:provider test-provider})
+
+(defmethod semantic.embedding/embedder-circuit-endpoint test-provider [_] test-endpoint)
+
+(def ^:private call-through* #'semantic.embedding/call-through-embedder-breaker)
+(def ^:private openai-compatible-get-embeddings-batch* #'semantic.embedding/openai-compatible-get-embeddings-batch)
+(def ^:private validate-embeddings!* #'semantic.embedding/validate-embeddings!)
+
+(defn- call-through [thunk & {:as opts}]
+ (apply call-through* thunk (mapcat identity (merge {:endpoint test-endpoint} opts))))
+
+(defn- circuit-state
+ ([] (circuit-state test-endpoint))
+ ([endpoint] (semantic.embedding/embedder-circuit-state endpoint)))
+
+(defn- boom [] (throw (ex-info "embedder down" {:kind :boom})))
+
+(deftest embedder-circuit-endpoint-is-a-provider-capability-test
+ (testing "providers without a remote circuit endpoint remain trusted"
+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled true]
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model
+ (constantly {:provider ::in-process})]
+ (is (nil? (semantic.embedding/embedder-circuit-state)))
+ (is (false? (semantic.embedding/embedder-circuit-untrusted?))))))
+ (testing "remote providers resolve the same endpoint their HTTP implementation uses"
+ (is (= "http://localhost:11434/api/embeddings"
+ (semantic.embedding/embedder-circuit-endpoint {:provider "ollama"})))
+ (mt/with-dynamic-fn-redefs
+ [semantic.settings/ee-embedding-service-base-url (constantly "https://embed.example/")
+ semantic.settings/ee-embedding-service-api-key (constantly "test-key")
+ llm.settings/ai-service-base-url (constantly nil)]
+ (is (= "https://embed.example/v1/embeddings"
+ (semantic.embedding/embedder-circuit-endpoint {:provider "ai-service"}))))
+ (mt/with-dynamic-fn-redefs
+ [semantic.settings/openai-api-base-url (constantly "https://openai.example")
+ semantic.settings/openai-api-key (constantly "test-key")]
+ (is (= "https://openai.example/v1/embeddings"
+ (semantic.embedding/embedder-circuit-endpoint {:provider "openai"}))))))
+
+(deftest ^:sequential request-failure-does-not-change-service-failure-history-test
+ (testing "request- and model-specific failures neither trip the circuit nor reset service-failure history"
+ (let [breaker (dh.cb/circuit-breaker {:failure-threshold 2 :success-threshold 1 :delay-ms 60000})]
+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})]
+ (is (thrown? Exception (call-through boom)))
+ (doseq [request-error [(ex-info "bad request" {:status 400})
+ (ex-info "model not found" {:status 404})
+ (ex-info "wrong dimensions" {:cause :embedder/unexpected-dimensions})]]
+ (is (thrown? Exception (call-through #(throw request-error)))))
+ (is (= :closed (circuit-state)))
+ (is (thrown? Exception (call-through boom)))
+ (is (= :open (circuit-state)))))))
+
+(deftest ^:sequential unexpected-dimensions-do-not-trip-breaker-test
+ (testing "dimensions are model-specific, so an otherwise valid response does not count as a service failure"
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {test-endpoint (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]
+ (is (=? {:cause :embedder/unexpected-dimensions
+ :expected-dimensions 1
+ :actual-dimensions 2}
+ (try
+ (call-through #(validate-embeddings!* [[0.0 1.0]] 1 1))
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ (ex-data e)))))
+ (is (= :closed (circuit-state))))))
+
+(deftest ^:sequential invalid-vector-values-trip-breaker-test
+ (testing "nonnumeric and non-finite values are invalid service responses"
+ (doseq [[label value] [["nonnumeric" ::not-a-number]
+ ["NaN" Double/NaN]
+ ["positive infinity" Double/POSITIVE_INFINITY]
+ ["negative infinity" Double/NEGATIVE_INFINITY]]]
+ (testing label
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {test-endpoint (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]
+ (is (instance? Exception
+ (try
+ (call-through #(validate-embeddings!* [[value]] 1 1))
+ nil
+ (catch Exception e
+ e))))
+ (is (= :open (circuit-state))))))))
+
+(deftest ^:sequential opens-after-threshold-and-fast-fails-test
+ (testing "consecutive failures trip the breaker; while open, calls fast-fail with the mapped 502 ex-info"
+ ;; A fresh breaker (short threshold, stays open) so the test is isolated from the process-wide default.
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {test-endpoint (dh.cb/circuit-breaker
+ {:failure-threshold 2 :success-threshold 1 :delay-ms 60000})})]
+ (dotimes [_ 2]
+ (is (thrown? Exception (call-through boom))))
+ (is (= :open (circuit-state)))
+ (is (=? {:cause :embedder/circuit-open :status 502}
+ (try (call-through (constantly :never)) nil
+ (catch clojure.lang.ExceptionInfo e (ex-data e))))))))
+
+(deftest ^:sequential circuit-open-fast-failure-does-not-log-request-error-test
+ (testing "the breaker transition reports the outage without an error log for every guarded request"
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {test-endpoint (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]
+ (is (thrown? Exception (call-through boom)))
+ (mt/with-log-messages-for-level [messages [metabase-enterprise.semantic-search.embedding :error]]
+ (is (=? {:cause :embedder/circuit-open :status 502}
+ (try
+ (openai-compatible-get-embeddings-batch*
+ {:provider "test"
+ :endpoint test-endpoint
+ :model-name "test-model"
+ :vector-dimensions 1
+ :texts ["text"]
+ :record-tokens? false})
+ nil
+ (catch clojure.lang.ExceptionInfo e
+ (ex-data e)))))
+ (is (empty? (messages)))))))
+
+(deftest ^:sequential throwable-releases-half-open-permit-test
+ (testing "a JVM-level failure cannot strand the breaker's only half-open trial permit"
+ (let [breaker (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 0})]
+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})]
+ (is (thrown? Exception (call-through boom)))
+ (is (thrown? AssertionError (call-through #(throw (AssertionError. "fatal")))))
+ (is (= :ok (call-through (constantly :ok))))
+ (is (= :closed (circuit-state)))))))
+
+(deftest ^:sequential interruption-releases-half-open-permit-test
+ (testing "an interruption propagates unchanged and cannot strand the breaker's half-open trial permit"
+ (let [breaker (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 0})
+ interrupted (InterruptedException. "cancelled")]
+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})]
+ (is (thrown? Exception (call-through boom)))
+ (is (identical? interrupted
+ (try
+ (call-through #(throw interrupted))
+ nil
+ (catch InterruptedException e
+ e))))
+ (is (= :ok (call-through (constantly :ok))))
+ (is (= :closed (circuit-state)))))))
+
+(deftest ^:sequential local-bookkeeping-failure-does-not-trip-breaker-test
+ (testing "a token-tracking failure propagates without being attributed to the embedding service"
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {"https://example.test/v1/embeddings"
+ (dh.cb/circuit-breaker
+ {:failure-threshold 1, :success-threshold 1, :delay-ms 60000})})]
+ (mt/with-dynamic-fn-redefs
+ [http/post (constantly {:body (str "{\"usage\":{\"total_tokens\":0},"
+ "\"data\":[{\"embedding\":\"AAAAAA==\"}]}")})
+ analytics/inc! (constantly nil)
+ token-tracking/record-tokens (fn [& _] (throw (ex-info "appdb unavailable" {})))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"appdb unavailable"
+ (#'semantic.embedding/openai-compatible-get-embeddings-batch
+ {:provider "openai"
+ :endpoint "https://example.test/v1/embeddings"
+ :api-key "test-key"
+ :model-name "test-model"
+ :vector-dimensions 1
+ :texts ["bird"]
+ :record-tokens? true
+ :snowplow? false})))
+ (is (= :closed (circuit-state "https://example.test/v1/embeddings")))))))
+
+(deftest ^:sequential malformed-response-trips-breaker-test
+ (testing "a successful HTTP response is not a circuit success until its vectors are validated"
+ (let [endpoint "https://example.test/v1/embeddings"]
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {endpoint (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]
+ (mt/with-dynamic-fn-redefs
+ [http/post (constantly {:body "{\"usage\":{},\"data\":[]}"})]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"unexpected number"
+ (#'semantic.embedding/openai-compatible-get-embeddings-batch
+ {:provider "openai"
+ :endpoint endpoint
+ :api-key "test-key"
+ :model-name "test-model"
+ :vector-dimensions 1
+ :texts ["bird"]
+ :record-tokens? false})))
+ (is (= :open (circuit-state endpoint))))))))
+
+(deftest ^:sequential endpoint-circuits-are-isolated-test
+ (let [endpoint-a "https://a.example/v1/embeddings"
+ endpoint-b "https://b.example/v1/embeddings"
+ breaker #(dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})]
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {endpoint-a (breaker), endpoint-b (breaker)})]
+ (is (thrown? Exception (call-through boom :endpoint endpoint-a)))
+ (is (= :open (circuit-state endpoint-a)))
+ (is (= :ok (call-through (constantly :ok) :endpoint endpoint-b)))
+ (is (= :closed (circuit-state endpoint-b))))))
+
+(deftest ^:sequential kill-switch-bypasses-breaker-test
+ (testing "with the breaker disabled the thunk runs directly: raw exception propagates, breaker untouched"
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {test-endpoint (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]
+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled false]
+ (is (=? {:kind :boom}
+ (try (call-through boom) nil (catch clojure.lang.ExceptionInfo e (ex-data e)))))
+ (is (= :closed (circuit-state)) "a bypassed breaker records no failures")))))
+
+(deftest ^:sequential probe-bypass-flag-bypasses-breaker-test
+ (testing "*bypass-circuit-breaker* runs the thunk directly, without consulting or tripping the breaker"
+ (with-redefs [semantic.embedding/embedder-circuit-breakers
+ (atom {test-endpoint (dh.cb/circuit-breaker
+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]
+ (binding [semantic.embedding/*bypass-circuit-breaker* true]
+ (is (thrown? clojure.lang.ExceptionInfo (call-through boom)))
+ (is (= :closed (circuit-state)))))))
+
+(deftest ^:sequential circuit-untrusted?-covers-half-open-test
+ (testing "embedder-circuit-untrusted? is true whenever the enabled breaker isn't closed -- both :open and
+ :half-open -- so the health verdict can't flap as the breaker cycles between them"
+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled true]
+ (doseq [state [:open :half-open]]
+ (mt/with-dynamic-fn-redefs [semantic.embedding/embedder-circuit-state (constantly state)]
+ (is (true? (semantic.embedding/embedder-circuit-untrusted?)) (str state " reads untrusted"))))
+ (mt/with-dynamic-fn-redefs [semantic.embedding/embedder-circuit-state (constantly :closed)]
+ (is (false? (semantic.embedding/embedder-circuit-untrusted?)) ":closed reads trusted"))))
+ (testing "disabled -> trusted regardless of raw state (calls bypass the breaker)"
+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled false]
+ (mt/with-dynamic-fn-redefs [semantic.embedding/embedder-circuit-state (constantly :half-open)]
+ (is (false? (semantic.embedding/embedder-circuit-untrusted?)))))))
+
+(deftest embedding-problem-schedules-idle-recovery-test
+ (let [recoveries (atom 0)]
+ (mt/with-dynamic-fn-redefs
+ [embedding-health/embedding-service-reachable? (constantly {:reachable? true, :error nil})
+ semantic.embedding/embedder-circuit-untrusted? (constantly true)
+ embedding-health/request-circuit-recovery! #(swap! recoveries inc)]
+ (is (string? (embedding-health/embedding-problem)))
+ (is (= 1 @recoveries)))))
+
+(deftest ^:sequential open-circuit-recovery-waits-for-the-trial-window-test
+ (let [breaker (dh.cb/circuit-breaker {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})
+ scheduled (atom nil)]
+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})
+ embedding-health/recovery-state (atom {:running? false, :last-start-ns nil})]
+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model (constantly test-embedding-model)]
+ (is (thrown? Exception (call-through boom)))
+ (is (= :open (circuit-state)))
+ (mt/with-dynamic-fn-redefs [embedding-health/schedule-recovery! #(reset! scheduled %)]
+ (embedding-health/request-circuit-recovery!)
+ (is (fn? @scheduled) "recovery is deferred until after the breaker permits a trial"))))))
+
+(deftest ^:sequential recovery-scheduling-failure-releases-claim-test
+ (let [state (atom {:running? false, :last-start-ns nil})]
+ (with-redefs [embedding-health/recovery-state state]
+ (mt/with-dynamic-fn-redefs
+ [semantic.embedding/embedder-circuit-untrusted? (constantly true)
+ embedding-health/schedule-recovery! (fn [_] (throw (ex-info "scheduler stopped" {})))]
+ (is (nil? (embedding-health/request-circuit-recovery!)))
+ (is (false? (:running? @state)))))))
+
+(deftest ^:sequential failed-idle-recovery-schedules-another-trial-test
+ (let [state (atom {:running? true, :last-start-ns 0})
+ retries (atom 0)]
+ (with-redefs [embedding-health/recovery-state state]
+ (mt/with-dynamic-fn-redefs
+ [semantic.embedding/embedder-circuit-untrusted? (constantly true)
+ embedding-health/run-recovery-probe! #(throw (ex-info "still down" {}))
+ embedding-health/request-circuit-recovery! #(swap! retries inc)]
+ (#'embedding-health/recover-circuit!)
+ (is (false? (:running? @state)))
+ (is (= 1 @retries))))))
+
+(deftest circuit-open-requests-idle-recovery-test
+ (let [requests (atom 0)]
+ (mt/with-dynamic-fn-redefs [embedding-health/request-circuit-recovery! #(swap! requests inc)]
+ (#'embedding-health/request-recovery-on-open! :half-open)
+ (#'embedding-health/request-recovery-on-open! :open)
+ (is (= 1 @requests)))))
+
+(deftest ^:sequential state-changes-run-serially-in-arrival-order-test
+ (testing "transitions queue through one agent: a later transition's hooks can't overtake an earlier
+ transition whose hook is still blocked (e.g. on a probe against a down service), so a slow
+ pre-recovery check can't persist its stale result after the recovery one"
+ (let [events (atom [])
+ entered (CountDownLatch. 1)
+ release (CountDownLatch. 1)
+ completed (CountDownLatch. 2)
+ hook (fn [state]
+ (when (= :half-open state)
+ (.countDown entered)
+ (.await release 5 TimeUnit/SECONDS))
+ (swap! events conj state)
+ (.countDown completed))]
+ (mt/with-dynamic-fn-redefs
+ [health-inspector/run-and-save-check! (constantly nil)]
+ (try
+ (swap! semantic.embedding/embedder-circuit-state-change-hooks conj hook)
+ (#'semantic.embedding/on-embedder-circuit-state-change! :half-open)
+ (is (.await entered 5 TimeUnit/SECONDS) "half-open hook did not start")
+ (#'semantic.embedding/on-embedder-circuit-state-change! :closed)
+ (is (= [] @events) ":closed stays queued behind the blocked :half-open run")
+ (.countDown release)
+ (is (.await completed 5 TimeUnit/SECONDS) "queued hooks did not finish")
+ (is (= [:half-open :closed] @events))
+ (finally
+ (.countDown release)
+ (swap! semantic.embedding/embedder-circuit-state-change-hooks disj hook)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
index eb29a846d83..c1b44b1b897 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj
@@ -331,28 +331,32 @@
"tag" "embedding_generation"}}]
events))))))))
-(deftest token-tracking-write-test
+(deftest ^:sequential token-tracking-write-test
(mt/with-premium-features #{:semantic-search}
(when (string? (not-empty (:mb-pgvector-db-url env/env)))
(doseq [provider ["openai" "ai-service"]]
(semantic.tu/with-test-db! {:mode :blank}
- (let [mock-embedding (repeat 1024 1.0)
- mock-response {:data [{:object "embedding"
- :embedding (encode-floats-to-base64 mock-embedding)
- :index 0}]
- :model "some-model"
- :usage {:prompt_tokens 1
- :total_tokens 13}}]
+ (let [encoded-embedding (encode-floats-to-base64 (repeat 1024 1.0))]
(with-redefs [semantic.settings/ee-embedding-provider (constantly provider)
semantic.settings/ee-embedding-model (constantly "mock-model")
semantic.settings/openai-api-key (constantly "xyz")
semantic.settings/openai-api-base-url (constantly "xyz")
semantic.settings/ee-embedding-service-base-url (constantly "http://mock-embedding-service")
semantic.settings/ee-embedding-service-api-key (constantly "mock-key")
- http/post (fn post-mock [_url & _options]
- {:status 200
- :headers {"Content-Type" "application/json"}
- :body (json/encode mock-response)})]
+ http/post (fn post-mock [_url {:keys [body]}]
+ (let [inputs (:input (json/decode body true))]
+ {:status 200
+ :headers {"Content-Type" "application/json"}
+ :body (json/encode
+ {:data (map-indexed
+ (fn [index _]
+ {:object "embedding"
+ :embedding encoded-embedding
+ :index index})
+ inputs)
+ :model "some-model"
+ :usage {:prompt_tokens 1
+ :total_tokens 13}})}))]
(let [pgvector (semantic.env/get-pgvector-datasource!)
index-metadata (semantic.env/get-index-metadata)
embedding-model (semantic.env/get-configured-embedding-model)