The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add embedder circuit breaker and recovery (#78296)
Chris Truter020b02932d83
diff --git a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj b/enterprise/backend/src/metabase_enterprise/embeddings/client.cljindex 607c81d3daf..bb6cf1a547d 100644--- a/enterprise/backend/src/metabase_enterprise/embeddings/client.clj+++ b/enterprise/backend/src/metabase_enterprise/embeddings/client.clj@@ -31,6 +31,14 @@(set! *warn-on-reflection* true)+(defn- search-embedding-model+ "Translate the neutral embedding descriptor's dimension key to semantic search's internal key."+ [embedding-model]+ (cond-> embedding-model+ (and (nil? (:vector-dimensions embedding-model))+ (some? (:model-dimensions embedding-model)))+ (assoc :vector-dimensions (:model-dimensions embedding-model))))+(defn get-embeddings-batch"Return a sequential collection of embedding vectors, in the same order as the input texts.@@ -41,4 +49,4 @@`opts` — optional keyword opts (e.g. `:type :doc`). Accepts alternating kwargs or a single trailing map;forwarded as kwargs into the multimethod, which destructures with `& {:as opts}`."[embedding-model texts & {:as opts}]- (apply semantic-search/get-embeddings-batch embedding-model texts (mapcat identity opts)))+ (apply semantic-search/get-embeddings-batch (search-embedding-model embedding-model) texts (mapcat identity opts)))diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.cljindex c3e06f03c80..fbf5524aeab 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding.clj@@ -2,6 +2,8 @@(:require[clj-http.client :as http][clojure.string :as str]+ [diehard.circuit-breaker :as dh.cb]+ [flatland.ordered.set :refer [ordered-set]][metabase-enterprise.semantic-search.models.token-tracking :as semantic.models.token-tracking][metabase-enterprise.semantic-search.settings :as semantic-settings][metabase.analytics-interface.core :as analytics]@@ -16,6 +18,7 @@(:import[com.knuddels.jtokkit Encodings][com.knuddels.jtokkit.api Encoding EncodingType]+ [dev.failsafe CircuitBreaker][java.net ConnectException]))(set! *warn-on-reflection* true)@@ -107,7 +110,7 @@(do(log/warn(format "Skipping text that exceeds maximum measure per batch: %s"- (str (subs text 0 10) "..."))+ (str (subs text 0 (min 10 (count text))) "...")){:measure text-measure :threshold threshold})acc)@@ -136,54 +139,267 @@(defn- dispatch-provider [embedding-model & _] (:provider embedding-model))(defmulti get-embedding- "Returns a single embedding vector for the given text"+ "Returns a single embedding vector for the given text.+ `opts` (kwargs, honoured by the production providers; ollama ignores them):+ - `:record-tokens?` — write a token-tracking row for the call+ - `:type` — what the embedding is for (`:query`/`:index`), recorded with the tokens+ - `:snowplow?` — ai-service only, default true; synthetic callers (e.g. the health probe) pass+ false so the call emits no token_usage event"{:arglists '([embedding-model text & opts])} dispatch-provider)(defmulti get-embeddings-batch- "Returns a sequential collection of embedding vectors, in the same order as the input texts."+ "Returns a sequential collection of embedding vectors, in the same order as the input texts.+ Takes the same `opts` as [[get-embedding]], minus `:snowplow?` (batch callers are all organic)."{:arglists '([embedding-model texts & opts])} dispatch-provider)+(defmulti embedder-circuit-endpoint+ "Return the remote endpoint that identifies `embedding-model`'s circuit, or nil when its provider does not+ use the remote-service circuit breaker."+ {:arglists '([embedding-model])} dispatch-provider)++(defmethod embedder-circuit-endpoint :default [_] nil)+(defmulti pull-model"If a model needs to be downloaded (which is the case for ollama), downloads it."{:arglists '([embedding-model])} dispatch-provider)+;;;; Embedding-service circuit breaker+;;;+;;; The embedding service is a remote HTTP dependency; when it's down, every semantic-search / NLQ query+;;; would otherwise re-attempt (and re-time-out) against it. A circuit breaker fails fast after repeated+;;; failures, and its state doubles as a health signal: an open breaker means real traffic is currently+;;; failing. State transitions surface the affected health checks immediately (see the listeners) rather+;;; than waiting for the daily job. Thresholds are fixed (the breaker is built once); the setting is a+;;; runtime kill switch checked per call.++(def ^:private embedder-circuit-breaker-failure-threshold+ "Consecutive embedding-service failures that trip the breaker open." 5)++(def ^:private embedder-circuit-breaker-success-threshold+ "Consecutive successes in half-open that close the breaker again." 2)++(def ^:private embedder-circuit-breaker-delay-ms+ "How long the breaker stays open before it allows a half-open trial call." 30000)++(def ^:private embedding-http-timeouts+ "clj-http timeout opts merged into every embedding-service request. Without an explicit :socket-timeout the+ client waits forever, so a blackholed service (accepts connections, never responds) would hang callers+ indefinitely -- and the breaker would never open, since only completed failures count toward it."+ {:connection-timeout 10000+ :socket-timeout 60000})++(defonce ^{:doc "Insertion-ordered set of `(fn [state])` hooks run on every breaker state change.+ Health namespaces `conj` a hook here (inverting the dep -- they require this module) to re-persist their+ embedder-dependent check on a transition, so an outage or recovery surfaces in minutes, not the next daily report.+ Ordered-set: `conj` is reload-idempotent and load-ordered, so the semantic hook clears the probe cache+ before the NLQ hook reads it. Register a var so a REPL redef takes effect live."}+ embedder-circuit-state-change-hooks+ (atom (ordered-set)))++(defonce ^:private embedder-circuit-hook-runner+ ;; One agent, so transitions run strictly in arrival order: concurrent futures would let a slow earlier+ ;; transition (e.g. its check blocked on a probe against a down service) persist a stale result AFTER a+ ;; later recovery transition's healthy one, leaving a pre-recovery "unreachable" row standing.+ ;; :continue so an unexpected action error can't wedge the agent for the process lifetime.+ (agent nil+ :error-mode :continue+ :error-handler (fn [_agent e] (log/error e "Embedder circuit hook runner errored"))))++(defn- on-embedder-circuit-state-change!+ "Run the registered [[embedder-circuit-state-change-hooks]] off the failsafe callback thread -- one+ transition at a time in arrival order, hooks in registration order, each isolated so one failing hook+ doesn't starve the rest."+ [state]+ ;; Log level tracks the resulting state's severity:+ ;; :open -> WARN (degradation)+ ;; :half-open -> INFO (probing)+ ;; :closed -> INFO (recovered)+ ;; Recovery is still captured level-independently -- the hooks below persist a health row per transition.+ (if (= :open state)+ (log/warn "Embedding service circuit breaker opened" {:state state})+ (log/info "Embedding service circuit breaker changed state" {:state state}))+ (send-off+ embedder-circuit-hook-runner+ (fn [_]+ (doseq [hook @embedder-circuit-state-change-hooks]+ (try+ (hook state)+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/error e "Embedder circuit state-change hook failed")))))))++(defn- new-embedder-circuit-breaker []+ (dh.cb/circuit-breaker+ {:failure-threshold embedder-circuit-breaker-failure-threshold+ :success-threshold embedder-circuit-breaker-success-threshold+ :delay-ms embedder-circuit-breaker-delay-ms+ :on-open (fn [_] (on-embedder-circuit-state-change! :open))+ :on-half-open (fn [_] (on-embedder-circuit-state-change! :half-open))+ :on-close (fn [_] (on-embedder-circuit-state-change! :closed))}))++(defonce ^:private embedder-circuit-breakers+ ;; Provider settings can differ between semantic search and data complexity. Keying by endpoint prevents+ ;; one unavailable service from fast-failing calls to an unrelated, healthy service.+ (atom {}))++(declare get-configured-model)++(defn- circuit-breaker-for+ ^CircuitBreaker [endpoint]+ (or (get @embedder-circuit-breakers endpoint)+ (get (swap! embedder-circuit-breakers+ #(if (contains? % endpoint)+ %+ (assoc % endpoint (new-embedder-circuit-breaker))))+ endpoint)))++(def ^:dynamic *bypass-circuit-breaker*+ "Bind true to run an embedding call without consulting or tripping the breaker.+ The health probe binds it so the probe stays an independent signal and can't flip the breaker from inside+ a state-change listener (which would recurse)."+ false)++(defn embedder-circuit-state+ "Return the configured embedder circuit's state (`:closed`, `:open`, or `:half-open`), or nil when the+ provider has no circuit endpoint. The optional `endpoint` arity inspects another endpoint's isolated circuit."+ ([]+ (embedder-circuit-state (embedder-circuit-endpoint (get-configured-model))))+ ([endpoint]+ (when endpoint+ (dh.cb/state (circuit-breaker-for endpoint)))))++(defn embedder-circuit-untrusted?+ "Whether the breaker is enabled and not closed -- i.e. open or half-open, so it doesn't yet trust the+ embedding service (open short-circuits calls; half-open is on a single trial).+ False when the breaker is disabled or the provider has no circuit endpoint."+ []+ (boolean+ (and (semantic-settings/semantic-search-embedder-circuit-breaker-enabled)+ (when-let [state (embedder-circuit-state)]+ (not= :closed state)))))++(def ^:private request-specific-statuses+ "HTTP statuses caused by a particular input, rather than the embedding service as a whole."+ #{400 404 413 422})++(defn- service-failure?+ [e]+ (let [data (ex-data e)+ cause (some-> e ex-cause ex-data)+ status (or (:status data) (:status cause))+ reason (or (:cause data) (:cause cause))]+ (and (not (contains? request-specific-statuses status))+ (not= :embedder/unexpected-dimensions reason))))++(defn- circuit-open-ex+ [endpoint]+ (ex-info "embedding service unavailable (circuit open)"+ (cond-> {:status 502, :cause :embedder/circuit-open}+ endpoint (assoc :endpoint endpoint))))++(defn- call-through-embedder-breaker+ "Run `thunk` under the embedder circuit breaker, unless it is bypassed (probe) or disabled (kill switch).+ Circuits are isolated by endpoint. Service-wide failures update the circuit; request- and model-specific+ failures propagate without changing its history. An open circuit throws a 502 `ex-info` with+ `:cause :embedder/circuit-open`."+ [thunk & {:keys [endpoint]}]+ (if (or (nil? endpoint)+ *bypass-circuit-breaker*+ (not (semantic-settings/semantic-search-embedder-circuit-breaker-enabled)))+ (thunk)+ (let [^CircuitBreaker breaker (circuit-breaker-for endpoint)]+ (when-not (dh.cb/allow-execution? breaker)+ (throw (circuit-open-ex endpoint)))+ (let [execution-state (dh.cb/state breaker)]+ (try+ (let [result (thunk)]+ (.recordSuccess breaker)+ result)+ (catch InterruptedException e+ (when (= :half-open execution-state)+ (.recordFailure breaker))+ (throw e))+ (catch Exception e+ (cond+ (service-failure? e) (.recordException breaker e)+ (= :half-open execution-state) (.recordSuccess breaker))+ (throw e))+ (catch Throwable t+ ;; Every half-open permit must record an outcome, even when the JVM-level failure itself should+ ;; propagate unchanged rather than contribute to the service-failure history.+ (when (= :half-open execution-state)+ (.recordFailure breaker))+ (throw t)))))))++(defn- validate-embeddings!+ [embeddings expected-count expected-dimensions]+ (when-not (= expected-count (count embeddings))+ (throw (ex-info "Embedding response contained an unexpected number of vectors"+ {:cause :embedder/invalid-response+ :expected-count expected-count+ :actual-count (count embeddings)})))+ (doseq [embedding embeddings]+ (when-not (= expected-dimensions (count embedding))+ (throw (ex-info "Embedding response contained a vector with unexpected dimensions"+ {:cause :embedder/unexpected-dimensions+ :expected-dimensions expected-dimensions+ :actual-dimensions (count embedding)})))+ (when-not (every? #(Double/isFinite (double %)) embedding)+ (throw (ex-info "Embedding response contained a non-finite value"+ {:cause :embedder/invalid-response}))))+ embeddings)+;;;; Ollama impl-(defn- ollama-get-embedding [model-name text]+(def ^:private ollama-embeddings-endpoint+ "http://localhost:11434/api/embeddings") ;; TODO: we should make the host configurable++(defmethod embedder-circuit-endpoint "ollama" [_] ollama-embeddings-endpoint)++(defn- ollama-get-embedding [model-name vector-dimensions text](try;; TODO count ollama tokens into :metabase-search/semantic-embedding-tokens?(log/debug "Generating Ollama embedding for text of length:" (count text))- (-> (http/post "http://localhost:11434/api/embeddings" ;; TODO: we should make the host configurable- {:headers {"Content-Type" "application/json"}- :body (json/encode {:model model-name- :prompt text})})- :body- (json/decode true)- :embedding)+ (let [embedding (-> (http/post ollama-embeddings-endpoint+ (merge embedding-http-timeouts+ {:headers {"Content-Type" "application/json"}+ :body (json/encode {:model model-name+ :prompt text})}))+ :body+ (json/decode true)+ :embedding)]+ (first (validate-embeddings! [embedding] 1 vector-dimensions)))(catch Exception e(log/error e "Failed to generate Ollama embedding for text of length:" (count text))(throw e))))-(defn- ollama-get-embeddings-batch [model-name texts]- ;; Ollama doesn't have a native batch API, so we fall back to individual calls- ;; No special batching needed for Ollama - just process all texts- (log/debug "Generating" (count texts) "Ollama embeddings (using individual calls)")- (mapv #(ollama-get-embedding model-name %) texts))-(defn- ollama-pull-model [model-name](try(log/debug "Pulling embedding model from Ollama...")(http/post "http://localhost:11434/api/pull" ;; TODO: make the host configurable- {:headers {"Content-Type" "application/json"}- :body (json/encode {:model model-name})})+ (merge embedding-http-timeouts+ {:headers {"Content-Type" "application/json"}+ :body (json/encode {:model model-name})}))(catch Exception e(log/error e "Failed to pull embedding model")(throw e))));; Ollama is not used in production. Token tracking is not implemented.-(defmethod get-embedding "ollama" [{:keys [model-name]} text & {:as _opts}] (ollama-get-embedding model-name text))-(defmethod get-embeddings-batch "ollama" [{:keys [model-name]} texts & {:as _opts}] (ollama-get-embeddings-batch model-name texts))-(defmethod pull-model "ollama" [{:keys [model-name]}] (ollama-pull-model model-name))+(defmethod get-embedding "ollama" [{:keys [model-name vector-dimensions]} text & {:as _opts}]+ (call-through-embedder-breaker #(ollama-get-embedding model-name vector-dimensions text)+ :endpoint ollama-embeddings-endpoint))++(defmethod get-embeddings-batch "ollama" [{:keys [model-name vector-dimensions]} texts & {:as _opts}]+ ;; Ollama doesn't have a native batch API, so we fall back to individual calls. Each call goes through the+ ;; breaker on its own, so an outage mid-batch fast-fails the remaining texts instead of timing out on each.+ (log/debug "Generating" (count texts) "Ollama embeddings (using individual calls)")+ (mapv (fn [text]+ (call-through-embedder-breaker #(ollama-get-embedding model-name vector-dimensions text)+ :endpoint ollama-embeddings-endpoint))+ texts))++(defmethod pull-model "ollama" [{:keys [model-name]}] (ollama-pull-model model-name));;;; OpenAI-compatible embedding service impl (shared by "ai-service" and "openai" providers)@@ -196,25 +412,28 @@(str/starts-with? model-name "text-embedding-3"))))(mu/defn- openai-compatible-get-embeddings-batch- "Call an OpenAI-compatible /v1/embeddings endpoint. Shared implementation for both- the `ai-service` and `openai` providers.+ "Call an OpenAI-compatible /v1/embeddings endpoint. The breaker guards only the remote request and+ response decoding; analytics and token persistence happen after it returns.`:provider` — label for analytics (e.g. \"ai-service\", \"openai\")`:endpoint` — full URL including /v1/embeddings`:api-key` — Bearer token. If empty ai service proxying is assumed and premium-embedding-token isused for authentication`:model-name` — model identifier sent in the request body+ `:vector-dimensions` — expected dimensions of each returned vector`:texts` — collection of input strings`:record-tokens?` — true writes a `semantic_search_token_tracking` row, false skips it.`:snowplow?` — optional; when true fires a Snowplow `token_usage` event`:extra-body` — optional; merged into the request body (e.g. `{:dimensions 1024}`)`:type` — optional; forwarded to the token-tracking row"- [{:keys [provider endpoint api-key model-name texts record-tokens? extra-body snowplow?] :as opts}+ [{:keys [provider endpoint api-key model-name vector-dimensions texts record-tokens? extra-body snowplow?]+ :as opts}:- [:map[:provider :string][:endpoint :string][:api-key {:optional true} [:maybe :string]][:model-name :string]+ [:vector-dimensions pos-int?][:texts [:sequential :string]][:record-tokens? :boolean][:snowplow? {:optional true} [:maybe :boolean]]@@ -222,24 +441,32 @@(try(log/debug (str "Calling " provider " embeddings API"){:endpoint endpoint :documents (count texts) :tokens (count-tokens-batch texts)})- (let [start-ms (u/start-timer)- {:keys [usage data]} (-> (http/post endpoint- {:headers- (merge {"Content-Type" "application/json"}- (if (and (empty? api-key)- (= "ai-service" provider))- {"x-metabase-instance-token"- (u/prog1 (premium-features/premium-embedding-token)- (when (nil? <>)- (throw (ex-info "Premium embedding token not set"- {:provider provider}))))}- {"Authorization" (str "Bearer " api-key)}))- :body (json/encode (merge {:model model-name- :input texts- :encoding_format "base64"}- extra-body))})- :body- (json/decode true))+ (let [headers (merge {"Content-Type" "application/json"}+ (if (and (empty? api-key) (= "ai-service" provider))+ {"x-metabase-instance-token"+ (u/prog1 (premium-features/premium-embedding-token)+ (when (nil? <>)+ (throw (ex-info "Premium embedding token not set"+ {:provider provider}))))}+ {"Authorization" (str "Bearer " api-key)}))+ request (merge embedding-http-timeouts+ {:headers headers+ :body (json/encode+ (merge {:model model-name+ :input texts+ :encoding_format "base64"}+ extra-body))})+ start-ms (u/start-timer)+ {:keys [usage embeddings]}+ (call-through-embedder-breaker+ #(let [{:keys [usage data]} (-> (http/post endpoint request)+ :body+ (json/decode true))]+ {:usage usage+ :embeddings (validate-embeddings! (decode-embeddings data)+ (count texts)+ vector-dimensions)})+ :endpoint endpoint)total-tokens (:total_tokens usage 0)prompt-tokens (:prompt_tokens usage total-tokens)](analytics/inc! :metabase-search/semantic-embedding-tokens@@ -259,15 +486,18 @@:tag "embedding_generation"}))(when record-tokens?(semantic.models.token-tracking/record-tokens model-name (:type opts) total-tokens))- (decode-embeddings data))+ embeddings)(catch ConnectException e(log/error e (str "Failed to connect to " provider) {:endpoint endpoint})(throw (ex-info (str provider " unavailable (connection refused)"){:status 502 :endpoint endpoint}e)))(catch Exception e- (log/error e (str provider " embeddings API call failed")- {:documents (count texts) :tokens (count-tokens-batch texts)})+ ;; The breaker transition already logs the outage once. Fast-failed calls while it remains open are+ ;; expected and can be frequent, so do not emit a redundant error for every guarded request.+ (when-not (= :embedder/circuit-open (:cause (ex-data e)))+ (log/error e (str "Failed to generate " provider " embeddings")+ {:documents (count texts) :tokens (count-tokens-batch texts)}))(throw e))));;;; Embedding-service provider@@ -296,27 +526,32 @@{:settings ["ee-embedding-service-base-url""ai-service-base-url"]}))))+(defmethod embedder-circuit-endpoint "ai-service" [_]+ (first (embedding-service-resolve-config!)))+(defmethod get-embedding "ai-service"- [{:keys [model-name]} text & {:keys [record-tokens? type]}]+ [{:keys [model-name vector-dimensions]} text & {:keys [record-tokens? type snowplow?] :or {snowplow? true}}](let [[endpoint api-key] (embedding-service-resolve-config!)](first (openai-compatible-get-embeddings-batch{:provider "ai-service":endpoint endpoint:api-key api-key:model-name model-name+ :vector-dimensions vector-dimensions:texts [text]- :snowplow? true+ :snowplow? snowplow?:record-tokens? record-tokens?:type type}))))(defmethod get-embeddings-batch "ai-service"- [{:keys [model-name]} texts & {:keys [record-tokens? type]}]+ [{:keys [model-name vector-dimensions]} texts & {:keys [record-tokens? type]}](let [[endpoint api-key] (embedding-service-resolve-config!)](openai-compatible-get-embeddings-batch{:provider "ai-service":endpoint endpoint:api-key api-key:model-name model-name+ :vector-dimensions vector-dimensions:texts texts:snowplow? true:record-tokens? record-tokens?@@ -335,6 +570,9 @@(throw (ex-info "OpenAI API key not configured" {:setting "llm-openai-api-key"})))[(str (semantic-settings/openai-api-base-url) "/v1/embeddings") api-key]))+(defmethod embedder-circuit-endpoint "openai" [_]+ (first (openai-resolve-config!)))+(defmethod get-embedding "openai"[embedding-model text & {:keys [record-tokens? type]}](let [[endpoint api-key] (openai-resolve-config!)]@@ -343,6 +581,7 @@:endpoint endpoint:api-key api-key:model-name (:model-name embedding-model)+ :vector-dimensions (:vector-dimensions embedding-model):texts [text]:record-tokens? record-tokens?:extra-body (when (supports-dimensions? embedding-model)@@ -357,6 +596,7 @@:endpoint endpoint:api-key api-key:model-name (:model-name embedding-model)+ :vector-dimensions (:vector-dimensions embedding-model):texts texts:record-tokens? record-tokens?:extra-body (when (supports-dimensions? embedding-model)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/embedding_health.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding_health.cljnew file mode 100644index 00000000000..f189f0436bf--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/embedding_health.clj@@ -0,0 +1,123 @@+(ns metabase-enterprise.semantic-search.embedding-health+ "Embedding-service liveness and circuit recovery shared by semantic search and entity retrieval."+ (:require+ [clojure.core.memoize :as memoize]+ [metabase-enterprise.semantic-search.embedding :as embedding]+ [metabase.util.log :as log])+ (:import+ (java.util.concurrent Executors ScheduledExecutorService ThreadFactory TimeUnit)))++(set! *warn-on-reflection* true)++(defn- probe-embedding-service+ []+ (try+ ;; This liveness signal must not mutate the breaker; recovery trials are scheduled separately below.+ (binding [embedding/*bypass-circuit-breaker* true]+ (embedding/get-embedding (embedding/get-configured-model)+ "health check"+ {:type :query, :record-tokens? false, :snowplow? false}))+ {:reachable? true, :error nil}+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/debug e "Embedding service health probe failed")+ {:reachable? false, :error (ex-message e)})))++(def ^:private embedding-service-reachable?*+ (memoize/ttl probe-embedding-service :ttl/threshold (* 60 1000)))++(defn embedding-service-reachable?+ "Probe the configured embedding service. Results are shared for a 60-second window."+ []+ (embedding-service-reachable?*))++(declare request-circuit-recovery!)++(defn embedding-problem+ "Return an embedding-service problem, or nil when the service and circuit are ready.+ A reachable probe schedules circuit recovery when an idle workload has not supplied trial calls."+ []+ (let [{:keys [reachable? error]} (embedding-service-reachable?)]+ (cond+ (not reachable?)+ (str "embedding service unreachable: " error)++ (embedding/embedder-circuit-untrusted?)+ (do+ (request-circuit-recovery!)+ "embedder circuit open (probe reachable; breaker still guarding calls)"))))++(def ^:private recovery-delay-ms 30000)++(def ^:private recovery-cooldown-ns+ (* recovery-delay-ms 1000000))++(defonce ^:private ^ScheduledExecutorService recovery-scheduler+ (Executors/newSingleThreadScheduledExecutor+ (reify ThreadFactory+ (newThread [_ runnable]+ (doto (Thread. runnable "embedding-circuit-recovery")+ (.setDaemon true))))))++(defonce ^:private recovery-state+ (atom {:running? false, :last-start-ns nil}))++(defn- claim-recovery! [now-ns]+ (loop []+ (let [{:keys [running? last-start-ns] :as state} @recovery-state]+ (cond+ running? false+ (and last-start-ns (< (- now-ns last-start-ns) recovery-cooldown-ns)) false+ (compare-and-set! recovery-state state {:running? true, :last-start-ns now-ns}) true+ :else (recur)))))++(defn- run-recovery-probe! []+ (embedding/get-embedding (embedding/get-configured-model)+ "health check"+ {:type :query, :record-tokens? false, :snowplow? false}))++(defn- recover-circuit! []+ (try+ ;; More than the current two-success threshold keeps this correct if that threshold is raised modestly.+ (loop [remaining 4]+ (when (and (pos? remaining) (embedding/embedder-circuit-untrusted?))+ (run-recovery-probe!)+ (recur (dec remaining))))+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/debug e "Embedding service circuit recovery probe failed"))+ (finally+ (swap! recovery-state assoc :running? false)))+ ;; A still-untrusted breaker needs another delayed trial even when no user traffic or health consumer runs.+ (when (embedding/embedder-circuit-untrusted?)+ (request-circuit-recovery!)))++(defn- schedule-recovery! [f]+ (.schedule recovery-scheduler ^Runnable f (long recovery-delay-ms) TimeUnit/MILLISECONDS))++(defn request-circuit-recovery!+ "Schedule a throttled breaker-controlled probe for the configured semantic-search embedder after the breaker's+ open delay. Semantic search and library retrieval share this embedder. Returns promptly."+ []+ (when (and (embedding/embedder-circuit-untrusted?)+ (claim-recovery! (System/nanoTime)))+ (try+ (schedule-recovery! recover-circuit!)+ (catch Exception e+ (swap! recovery-state assoc :running? false)+ (log/error e "Failed to schedule embedding service circuit recovery"))))+ nil)++(defn- clear-probe-cache-on-recovery! [state]+ (when (= :closed state)+ (memoize/memo-clear! embedding-service-reachable?*)))++(defn- request-recovery-on-open! [state]+ (when (= :open state)+ (request-circuit-recovery!)))++;; Register the shared cache hook before either consumer registers its health-row hook.+(swap! embedding/embedder-circuit-state-change-hooks conj #'clear-probe-cache-on-recovery!)+(swap! embedding/embedder-circuit-state-change-hooks conj #'request-recovery-on-open!)diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/init.cljindex 330cec96293..07908499c6c 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/init.clj@@ -1,5 +1,6 @@(ns metabase-enterprise.semantic-search.init(:require+ [metabase-enterprise.semantic-search.embedding-health][metabase-enterprise.semantic-search.events][metabase-enterprise.semantic-search.settings][metabase-enterprise.semantic-search.task.index-cleanup]diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.cljindex 34a90d8a90e..c4abc0798aa 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/settings.clj@@ -85,6 +85,17 @@:export? false:doc false)+(defsetting semantic-search-embedder-circuit-breaker-enabled+ (deferred-tru+ (str "Wrap embedding-service calls in a circuit breaker that fails fast after repeated failures. "+ "Runtime kill switch; the breaker thresholds are fixed."))+ :type :boolean+ :default true+ :encryption :no+ :export? false+ :visibility :internal+ :doc false)+(defsetting openai-max-tokens-per-batch(deferred-tru "The maximum number of tokens sent in a single embedding API call."):type :integerdiff --git a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.cljindex c8c8037f824..494c0287436 100644--- a/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_complexity_score/complexity_test.clj@@ -31,6 +31,10 @@(def ^:private test-entity-ids (atom 0))+(defmethod semantic-search/get-embeddings-batch ::dimension-translation-test+ [embedding-model _texts & _opts]+ [embedding-model])+(defn- entity"Build a fake entity map for scoring tests. Uses a monotonically-increasing counter for `:id`so every test entity is distinct (and so test failures print stable, readable ids)."@@ -744,6 +748,13 @@(embedder [{:id 1 :name "orders" :kind :table}])(is (false? (:record-tokens? @captured)))))))+(deftest ^:parallel embeddings-client-translates-neutral-dimension-key-test+ (let [[translated]+ (embeddings/get-embeddings-batch+ {:provider ::dimension-translation-test, :model-name "fake", :model-dimensions 384}+ ["orders"])]+ (is (= 384 (:vector-dimensions translated)))))+(deftest ^:sequential provider-embedder-splits-names-before-calling-provider-test(testing "provider-embedder splits names on _, -, ., and camelCase before sending to get-embeddings-batch"(let [captured (atom nil)diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.cljnew file mode 100644index 00000000000..9c09f1afc6e--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_circuit_breaker_test.clj@@ -0,0 +1,333 @@+(ns metabase-enterprise.semantic-search.embedding-circuit-breaker-test+ (:require+ [clj-http.client :as http]+ [clojure.test :refer :all]+ [diehard.circuit-breaker :as dh.cb]+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]+ [metabase-enterprise.semantic-search.models.token-tracking :as token-tracking]+ [metabase-enterprise.semantic-search.settings :as semantic.settings]+ [metabase.analytics-interface.core :as analytics]+ [metabase.health-inspector.core :as health-inspector]+ [metabase.llm.settings :as llm.settings]+ [metabase.test :as mt])+ (:import+ (java.util.concurrent CountDownLatch TimeUnit)))++(set! *warn-on-reflection* true)++(def ^:private test-endpoint "https://circuit-test.example/v1/embeddings")+(def ^:private test-provider ::circuit-test)+(def ^:private test-embedding-model {:provider test-provider})++(defmethod semantic.embedding/embedder-circuit-endpoint test-provider [_] test-endpoint)++(def ^:private call-through* #'semantic.embedding/call-through-embedder-breaker)+(def ^:private openai-compatible-get-embeddings-batch* #'semantic.embedding/openai-compatible-get-embeddings-batch)+(def ^:private validate-embeddings!* #'semantic.embedding/validate-embeddings!)++(defn- call-through [thunk & {:as opts}]+ (apply call-through* thunk (mapcat identity (merge {:endpoint test-endpoint} opts))))++(defn- circuit-state+ ([] (circuit-state test-endpoint))+ ([endpoint] (semantic.embedding/embedder-circuit-state endpoint)))++(defn- boom [] (throw (ex-info "embedder down" {:kind :boom})))++(deftest embedder-circuit-endpoint-is-a-provider-capability-test+ (testing "providers without a remote circuit endpoint remain trusted"+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled true]+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model+ (constantly {:provider ::in-process})]+ (is (nil? (semantic.embedding/embedder-circuit-state)))+ (is (false? (semantic.embedding/embedder-circuit-untrusted?))))))+ (testing "remote providers resolve the same endpoint their HTTP implementation uses"+ (is (= "http://localhost:11434/api/embeddings"+ (semantic.embedding/embedder-circuit-endpoint {:provider "ollama"})))+ (mt/with-dynamic-fn-redefs+ [semantic.settings/ee-embedding-service-base-url (constantly "https://embed.example/")+ semantic.settings/ee-embedding-service-api-key (constantly "test-key")+ llm.settings/ai-service-base-url (constantly nil)]+ (is (= "https://embed.example/v1/embeddings"+ (semantic.embedding/embedder-circuit-endpoint {:provider "ai-service"}))))+ (mt/with-dynamic-fn-redefs+ [semantic.settings/openai-api-base-url (constantly "https://openai.example")+ semantic.settings/openai-api-key (constantly "test-key")]+ (is (= "https://openai.example/v1/embeddings"+ (semantic.embedding/embedder-circuit-endpoint {:provider "openai"}))))))++(deftest ^:sequential request-failure-does-not-change-service-failure-history-test+ (testing "request- and model-specific failures neither trip the circuit nor reset service-failure history"+ (let [breaker (dh.cb/circuit-breaker {:failure-threshold 2 :success-threshold 1 :delay-ms 60000})]+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})]+ (is (thrown? Exception (call-through boom)))+ (doseq [request-error [(ex-info "bad request" {:status 400})+ (ex-info "model not found" {:status 404})+ (ex-info "wrong dimensions" {:cause :embedder/unexpected-dimensions})]]+ (is (thrown? Exception (call-through #(throw request-error)))))+ (is (= :closed (circuit-state)))+ (is (thrown? Exception (call-through boom)))+ (is (= :open (circuit-state)))))))++(deftest ^:sequential unexpected-dimensions-do-not-trip-breaker-test+ (testing "dimensions are model-specific, so an otherwise valid response does not count as a service failure"+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {test-endpoint (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]+ (is (=? {:cause :embedder/unexpected-dimensions+ :expected-dimensions 1+ :actual-dimensions 2}+ (try+ (call-through #(validate-embeddings!* [[0.0 1.0]] 1 1))+ nil+ (catch clojure.lang.ExceptionInfo e+ (ex-data e)))))+ (is (= :closed (circuit-state))))))++(deftest ^:sequential invalid-vector-values-trip-breaker-test+ (testing "nonnumeric and non-finite values are invalid service responses"+ (doseq [[label value] [["nonnumeric" ::not-a-number]+ ["NaN" Double/NaN]+ ["positive infinity" Double/POSITIVE_INFINITY]+ ["negative infinity" Double/NEGATIVE_INFINITY]]]+ (testing label+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {test-endpoint (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]+ (is (instance? Exception+ (try+ (call-through #(validate-embeddings!* [[value]] 1 1))+ nil+ (catch Exception e+ e))))+ (is (= :open (circuit-state))))))))++(deftest ^:sequential opens-after-threshold-and-fast-fails-test+ (testing "consecutive failures trip the breaker; while open, calls fast-fail with the mapped 502 ex-info"+ ;; A fresh breaker (short threshold, stays open) so the test is isolated from the process-wide default.+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {test-endpoint (dh.cb/circuit-breaker+ {:failure-threshold 2 :success-threshold 1 :delay-ms 60000})})]+ (dotimes [_ 2]+ (is (thrown? Exception (call-through boom))))+ (is (= :open (circuit-state)))+ (is (=? {:cause :embedder/circuit-open :status 502}+ (try (call-through (constantly :never)) nil+ (catch clojure.lang.ExceptionInfo e (ex-data e))))))))++(deftest ^:sequential circuit-open-fast-failure-does-not-log-request-error-test+ (testing "the breaker transition reports the outage without an error log for every guarded request"+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {test-endpoint (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]+ (is (thrown? Exception (call-through boom)))+ (mt/with-log-messages-for-level [messages [metabase-enterprise.semantic-search.embedding :error]]+ (is (=? {:cause :embedder/circuit-open :status 502}+ (try+ (openai-compatible-get-embeddings-batch*+ {:provider "test"+ :endpoint test-endpoint+ :model-name "test-model"+ :vector-dimensions 1+ :texts ["text"]+ :record-tokens? false})+ nil+ (catch clojure.lang.ExceptionInfo e+ (ex-data e)))))+ (is (empty? (messages)))))))++(deftest ^:sequential throwable-releases-half-open-permit-test+ (testing "a JVM-level failure cannot strand the breaker's only half-open trial permit"+ (let [breaker (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 0})]+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})]+ (is (thrown? Exception (call-through boom)))+ (is (thrown? AssertionError (call-through #(throw (AssertionError. "fatal")))))+ (is (= :ok (call-through (constantly :ok))))+ (is (= :closed (circuit-state)))))))++(deftest ^:sequential interruption-releases-half-open-permit-test+ (testing "an interruption propagates unchanged and cannot strand the breaker's half-open trial permit"+ (let [breaker (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 0})+ interrupted (InterruptedException. "cancelled")]+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})]+ (is (thrown? Exception (call-through boom)))+ (is (identical? interrupted+ (try+ (call-through #(throw interrupted))+ nil+ (catch InterruptedException e+ e))))+ (is (= :ok (call-through (constantly :ok))))+ (is (= :closed (circuit-state)))))))++(deftest ^:sequential local-bookkeeping-failure-does-not-trip-breaker-test+ (testing "a token-tracking failure propagates without being attributed to the embedding service"+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {"https://example.test/v1/embeddings"+ (dh.cb/circuit-breaker+ {:failure-threshold 1, :success-threshold 1, :delay-ms 60000})})]+ (mt/with-dynamic-fn-redefs+ [http/post (constantly {:body (str "{\"usage\":{\"total_tokens\":0},"+ "\"data\":[{\"embedding\":\"AAAAAA==\"}]}")})+ analytics/inc! (constantly nil)+ token-tracking/record-tokens (fn [& _] (throw (ex-info "appdb unavailable" {})))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"appdb unavailable"+ (#'semantic.embedding/openai-compatible-get-embeddings-batch+ {:provider "openai"+ :endpoint "https://example.test/v1/embeddings"+ :api-key "test-key"+ :model-name "test-model"+ :vector-dimensions 1+ :texts ["bird"]+ :record-tokens? true+ :snowplow? false})))+ (is (= :closed (circuit-state "https://example.test/v1/embeddings")))))))++(deftest ^:sequential malformed-response-trips-breaker-test+ (testing "a successful HTTP response is not a circuit success until its vectors are validated"+ (let [endpoint "https://example.test/v1/embeddings"]+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {endpoint (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]+ (mt/with-dynamic-fn-redefs+ [http/post (constantly {:body "{\"usage\":{},\"data\":[]}"})]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"unexpected number"+ (#'semantic.embedding/openai-compatible-get-embeddings-batch+ {:provider "openai"+ :endpoint endpoint+ :api-key "test-key"+ :model-name "test-model"+ :vector-dimensions 1+ :texts ["bird"]+ :record-tokens? false})))+ (is (= :open (circuit-state endpoint))))))))++(deftest ^:sequential endpoint-circuits-are-isolated-test+ (let [endpoint-a "https://a.example/v1/embeddings"+ endpoint-b "https://b.example/v1/embeddings"+ breaker #(dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})]+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {endpoint-a (breaker), endpoint-b (breaker)})]+ (is (thrown? Exception (call-through boom :endpoint endpoint-a)))+ (is (= :open (circuit-state endpoint-a)))+ (is (= :ok (call-through (constantly :ok) :endpoint endpoint-b)))+ (is (= :closed (circuit-state endpoint-b))))))++(deftest ^:sequential kill-switch-bypasses-breaker-test+ (testing "with the breaker disabled the thunk runs directly: raw exception propagates, breaker untouched"+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {test-endpoint (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled false]+ (is (=? {:kind :boom}+ (try (call-through boom) nil (catch clojure.lang.ExceptionInfo e (ex-data e)))))+ (is (= :closed (circuit-state)) "a bypassed breaker records no failures")))))++(deftest ^:sequential probe-bypass-flag-bypasses-breaker-test+ (testing "*bypass-circuit-breaker* runs the thunk directly, without consulting or tripping the breaker"+ (with-redefs [semantic.embedding/embedder-circuit-breakers+ (atom {test-endpoint (dh.cb/circuit-breaker+ {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})})]+ (binding [semantic.embedding/*bypass-circuit-breaker* true]+ (is (thrown? clojure.lang.ExceptionInfo (call-through boom)))+ (is (= :closed (circuit-state)))))))++(deftest ^:sequential circuit-untrusted?-covers-half-open-test+ (testing "embedder-circuit-untrusted? is true whenever the enabled breaker isn't closed -- both :open and+ :half-open -- so the health verdict can't flap as the breaker cycles between them"+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled true]+ (doseq [state [:open :half-open]]+ (mt/with-dynamic-fn-redefs [semantic.embedding/embedder-circuit-state (constantly state)]+ (is (true? (semantic.embedding/embedder-circuit-untrusted?)) (str state " reads untrusted"))))+ (mt/with-dynamic-fn-redefs [semantic.embedding/embedder-circuit-state (constantly :closed)]+ (is (false? (semantic.embedding/embedder-circuit-untrusted?)) ":closed reads trusted"))))+ (testing "disabled -> trusted regardless of raw state (calls bypass the breaker)"+ (mt/with-temporary-setting-values [semantic-search-embedder-circuit-breaker-enabled false]+ (mt/with-dynamic-fn-redefs [semantic.embedding/embedder-circuit-state (constantly :half-open)]+ (is (false? (semantic.embedding/embedder-circuit-untrusted?)))))))++(deftest embedding-problem-schedules-idle-recovery-test+ (let [recoveries (atom 0)]+ (mt/with-dynamic-fn-redefs+ [embedding-health/embedding-service-reachable? (constantly {:reachable? true, :error nil})+ semantic.embedding/embedder-circuit-untrusted? (constantly true)+ embedding-health/request-circuit-recovery! #(swap! recoveries inc)]+ (is (string? (embedding-health/embedding-problem)))+ (is (= 1 @recoveries)))))++(deftest ^:sequential open-circuit-recovery-waits-for-the-trial-window-test+ (let [breaker (dh.cb/circuit-breaker {:failure-threshold 1 :success-threshold 1 :delay-ms 60000})+ scheduled (atom nil)]+ (with-redefs [semantic.embedding/embedder-circuit-breakers (atom {test-endpoint breaker})+ embedding-health/recovery-state (atom {:running? false, :last-start-ns nil})]+ (mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model (constantly test-embedding-model)]+ (is (thrown? Exception (call-through boom)))+ (is (= :open (circuit-state)))+ (mt/with-dynamic-fn-redefs [embedding-health/schedule-recovery! #(reset! scheduled %)]+ (embedding-health/request-circuit-recovery!)+ (is (fn? @scheduled) "recovery is deferred until after the breaker permits a trial"))))))++(deftest ^:sequential recovery-scheduling-failure-releases-claim-test+ (let [state (atom {:running? false, :last-start-ns nil})]+ (with-redefs [embedding-health/recovery-state state]+ (mt/with-dynamic-fn-redefs+ [semantic.embedding/embedder-circuit-untrusted? (constantly true)+ embedding-health/schedule-recovery! (fn [_] (throw (ex-info "scheduler stopped" {})))]+ (is (nil? (embedding-health/request-circuit-recovery!)))+ (is (false? (:running? @state)))))))++(deftest ^:sequential failed-idle-recovery-schedules-another-trial-test+ (let [state (atom {:running? true, :last-start-ns 0})+ retries (atom 0)]+ (with-redefs [embedding-health/recovery-state state]+ (mt/with-dynamic-fn-redefs+ [semantic.embedding/embedder-circuit-untrusted? (constantly true)+ embedding-health/run-recovery-probe! #(throw (ex-info "still down" {}))+ embedding-health/request-circuit-recovery! #(swap! retries inc)]+ (#'embedding-health/recover-circuit!)+ (is (false? (:running? @state)))+ (is (= 1 @retries))))))++(deftest circuit-open-requests-idle-recovery-test+ (let [requests (atom 0)]+ (mt/with-dynamic-fn-redefs [embedding-health/request-circuit-recovery! #(swap! requests inc)]+ (#'embedding-health/request-recovery-on-open! :half-open)+ (#'embedding-health/request-recovery-on-open! :open)+ (is (= 1 @requests)))))++(deftest ^:sequential state-changes-run-serially-in-arrival-order-test+ (testing "transitions queue through one agent: a later transition's hooks can't overtake an earlier+ transition whose hook is still blocked (e.g. on a probe against a down service), so a slow+ pre-recovery check can't persist its stale result after the recovery one"+ (let [events (atom [])+ entered (CountDownLatch. 1)+ release (CountDownLatch. 1)+ completed (CountDownLatch. 2)+ hook (fn [state]+ (when (= :half-open state)+ (.countDown entered)+ (.await release 5 TimeUnit/SECONDS))+ (swap! events conj state)+ (.countDown completed))]+ (mt/with-dynamic-fn-redefs+ [health-inspector/run-and-save-check! (constantly nil)]+ (try+ (swap! semantic.embedding/embedder-circuit-state-change-hooks conj hook)+ (#'semantic.embedding/on-embedder-circuit-state-change! :half-open)+ (is (.await entered 5 TimeUnit/SECONDS) "half-open hook did not start")+ (#'semantic.embedding/on-embedder-circuit-state-change! :closed)+ (is (= [] @events) ":closed stays queued behind the blocked :half-open run")+ (.countDown release)+ (is (.await completed 5 TimeUnit/SECONDS) "queued hooks did not finish")+ (is (= [:half-open :closed] @events))+ (finally+ (.countDown release)+ (swap! semantic.embedding/embedder-circuit-state-change-hooks disj hook)))))))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.cljindex eb29a846d83..c1b44b1b897 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/embedding_test.clj@@ -331,28 +331,32 @@"tag" "embedding_generation"}}]events))))))))-(deftest token-tracking-write-test+(deftest ^:sequential token-tracking-write-test(mt/with-premium-features #{:semantic-search}(when (string? (not-empty (:mb-pgvector-db-url env/env)))(doseq [provider ["openai" "ai-service"]](semantic.tu/with-test-db! {:mode :blank}- (let [mock-embedding (repeat 1024 1.0)- mock-response {:data [{:object "embedding"- :embedding (encode-floats-to-base64 mock-embedding)- :index 0}]- :model "some-model"- :usage {:prompt_tokens 1- :total_tokens 13}}]+ (let [encoded-embedding (encode-floats-to-base64 (repeat 1024 1.0))](with-redefs [semantic.settings/ee-embedding-provider (constantly provider)semantic.settings/ee-embedding-model (constantly "mock-model")semantic.settings/openai-api-key (constantly "xyz")semantic.settings/openai-api-base-url (constantly "xyz")semantic.settings/ee-embedding-service-base-url (constantly "http://mock-embedding-service")semantic.settings/ee-embedding-service-api-key (constantly "mock-key")- http/post (fn post-mock [_url & _options]- {:status 200- :headers {"Content-Type" "application/json"}- :body (json/encode mock-response)})]+ http/post (fn post-mock [_url {:keys [body]}]+ (let [inputs (:input (json/decode body true))]+ {:status 200+ :headers {"Content-Type" "application/json"}+ :body (json/encode+ {:data (map-indexed+ (fn [index _]+ {:object "embedding"+ :embedding encoded-embedding+ :index index})+ inputs)+ :model "some-model"+ :usage {:prompt_tokens 1+ :total_tokens 13}})}))](let [pgvector (semantic.env/get-pgvector-datasource!)index-metadata (semantic.env/get-index-metadata)embedding-model (semantic.env/get-configured-embedding-model)