The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Refresh the premium token on a per-instance job instead of on read (#85297)
John Swansone04948d09c77
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 9235255485d..905c656cadf 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2047,7 +2047,6 @@app-dbdependenciessettings- startuptasktracingutildiff --git a/src/metabase/core/core.clj b/src/metabase/core/core.cljindex 4ae91449cbc..e499ae65a03 100644--- a/src/metabase/core/core.clj+++ b/src/metabase/core/core.clj@@ -25,7 +25,7 @@[metabase.notification.core :as notification][metabase.oauth-server.api :as oauth-server.api][metabase.plugins.core :as plugins]- [metabase.premium-features.core :refer [defenterprise]]+ [metabase.premium-features.core :as premium-features :refer [defenterprise]][metabase.sample-data.core :as sample-data][metabase.search.embeddings.startup :as embeddings.startup][metabase.server.core :as server]@@ -108,6 +108,7 @@[](log/info "Metabase Shutting Down ...")(queue/stop-listeners!)+ (premium-features/stop-refresher!)(startup/run-shutdown-logic!)(task/stop-scheduler!)(server/stop-web-server!)@@ -205,6 +206,9 @@;; negatives, but for now there's not much we can do(mdb/setup-db! :create-sample-content? (not config/is-test?))(mdb/encrypt-plaintext-columns!)+ ;; fetch the premium token status now, so that everything below sees this instance's features+ (premium-features/refresh-at-startup!)+ (premium-features/start-refresher!);; Disable read-only mode if its on during startup.;; This can happen if a cloud migration process dies during h2 dump.(when (cloud-migration/read-only-mode)diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.cljindex ccb93850eb0..791125584a0 100644--- a/src/metabase/premium_features/core.clj+++ b/src/metabase/premium_features/core.clj@@ -38,6 +38,9 @@TokenStatusclear-cache!clear-local-cache!+ refresh-at-startup!+ start-refresher!+ stop-refresher!token-check-urltransform-metered-astransform-stats]diff --git a/src/metabase/premium_features/db.clj b/src/metabase/premium_features/db.cljindex a9375e3c0f2..d67c8c5707c 100644--- a/src/metabase/premium_features/db.clj+++ b/src/metabase/premium_features/db.clj@@ -34,8 +34,3 @@(t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash:token_status_hash token-status-hash:updated_at updated-at}))--(mu/defn delete-token-status-cache!- "Delete every cached token status."- []- (t2/delete! :model/PremiumFeaturesCache))diff --git a/src/metabase/premium_features/init.clj b/src/metabase/premium_features/init.cljindex ba462c880e2..3708aa217b7 100644--- a/src/metabase/premium_features/init.clj+++ b/src/metabase/premium_features/init.clj@@ -2,5 +2,4 @@(:require[metabase.premium-features.core][metabase.premium-features.settings]- [metabase.premium-features.task.clear-token-cache][metabase.premium-features.task.send-metering]))diff --git a/src/metabase/premium_features/rest/api.clj b/src/metabase/premium_features/rest/api.cljindex 53d6a2e03db..9214af4c748 100644--- a/src/metabase/premium_features/rest/api.clj+++ b/src/metabase/premium_features/rest/api.clj@@ -51,15 +51,20 @@#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}(api.macros/defendpoint :post "/token/refresh"- "Clear all token caches and re-check the premium features token against the MetaStore.- Returns the fresh token status. Useful for the frontend after a purchase so that new features- are recognized without waiting for the cache TTL."+ "Re-check the premium features token against the MetaStore and return the fresh token status. Useful for the+ frontend after a purchase so that new features are recognized without waiting for the next scheduled refresh.++ When the check fails, returns an unvalidated status whose `:error-details` says why, so the caller knows the check+ failed. The status this instance holds is left as it was, so feature checks keep using it."[]- (token-check/clear-cache!)- (invalidate-llm-proxy-token-cache!)- (-> (api/check-404 (premium-features/token-status))- response/response- (assoc-in [:mb/cookies :cookie/premium-features-cache-timestamp] true)))+ (let [status (try+ (token-check/refresh-now!)+ (catch Throwable e+ (token-check/unvalidated-status (ex-message e))))]+ (invalidate-llm-proxy-token-cache!)+ (-> (api/check-404 status)+ response/response+ (assoc-in [:mb/cookies :cookie/premium-features-cache-timestamp] true))))(def ^{:arglists '([request respond raise])} routes"`/api/premium-features` routes."diff --git a/src/metabase/premium_features/task/clear_token_cache.clj b/src/metabase/premium_features/task/clear_token_cache.cljdeleted file mode 100644index a1fd658e9b9..00000000000--- a/src/metabase/premium_features/task/clear_token_cache.clj+++ /dev/null@@ -1,9 +0,0 @@-(ns metabase.premium-features.task.clear-token-cache- "Clears the premium features token cache on startup so the first token check always hits MetaStore fresh.- Without this, the cache table (which survives restarts) could serve stale feature data."- (:require- [metabase.premium-features.token-check :as token-check]- [metabase.startup.core :as startup]))--(defmethod startup/def-startup-logic! ::clear-token-cache [_]- (token-check/clear-cache!))diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex 454866b28c3..206d947b797 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -9,7 +9,6 @@[buddy.core.codecs :as codecs][buddy.core.hash :as buddy-hash][clj-http.client :as http]- [clojure.core.memoize :as memoize][clojure.string :as str][diehard.circuit-breaker :as dh.cb][diehard.core :as dh]@@ -31,10 +30,13 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms]+ [metabase.util.retry :as retry][metabase.util.string :as u.str]- [potemkin.types :as p][toucan2.connection :as t2.conn]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (dev.failsafe CircuitBreakerOpenException FailsafeException TimeoutExceededException)+ (java.util.concurrent Executors RejectedExecutionException ScheduledExecutorService ThreadFactory TimeUnit)))(set! *warn-on-reflection* true)@@ -234,17 +236,13 @@(log/infof "Checking with the MetaStore to see whether token '%s' is valid..." (u.str/mask token))(let [{:keys [body status] :as resp} (http-fetch base-url token site-uuid)](if (or (http/success? resp) (<= 400 status 499))- (do (when (http/success? resp)- (analytics/inc! :metabase-token-check/attempt {:status :success}))- (or (some-> (parse-status-body body) canonical)- ;; a 2xx/4xx whose body is missing or undecodable (a proxy's or WAF's error page, a stripped- ;; body) carries no verdict from the store, so it is transient like a network error: not cached,- ;; retried under the breaker, and the previous canonical status stands meanwhile- (throw (ex-info "Token validation provided no response." {:status status}))))- ;; exceptions are not cached.- (do (analytics/inc! :metabase-token-check/attempt {:status :failure})- (throw (ex-info "An unknown error occurred when validating token." {:status status- :body body}))))))+ (let [decoded (or (some-> (parse-status-body body) canonical)+ (throw (ex-info "Token validation provided no response." {:status status})))]+ (when (http/success? resp)+ (analytics/inc! :metabase-token-check/attempt {:status :success}))+ decoded)+ (throw (ex-info "An unknown error occurred when validating token." {:status status+ :body body})))))(defn- metering-url[token base-url]@@ -327,67 +325,6 @@:status "invalid":error-details (trs "Token should be a valid 64 hexadecimal character token or an airgap token.")}))))-#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}-(def ^:dynamic *token-check-happening* "Var to prevent recursive calls to `fetch-token-status`" false)--(p/defprotocol+ TokenChecker- "Protocol for checking tokens with cache management."- (-check-token [this token]- "Check a token and return TokenStatus map. May throw exceptions on failure.")- (-clear-cache! [this]- "Clear any caches in this checker and any wrapped checkers, including the shared DB cache.- Returns nil. Implementations should delegate to wrapped checkers.")- (-clear-local-cache! [this]- "Clear only the in-process memoization in this checker and any wrapped checkers. The shared DB cache and the- DB-hash-validated local cache are left alone, so the next check re-validates against the DB and only refetches- from the MetaStore if another instance really did get a different token status. Safe to trigger from untrusted- input. Returns nil."))--(def store-and-airgap-token-checker- "Creates a basic token checker that handles HTTP requests and airgap tokens.- No caching, no retries, no grace period - just the core logic."- (reify TokenChecker- (-check-token [_ token]- (decode-token* token))- (-clear-cache! [_] nil)- (-clear-local-cache! [_] nil)))--(defn circuit-breaker-token-checker- "Wraps a token checker with circuit breaker and timeout logic."- [token-checker {:keys [circuit-breaker timeout-ms lock]- :or {lock (Object.)}}]- (let [breaker (dh.cb/circuit-breaker circuit-breaker)]- (reify TokenChecker- (-check-token [_ token]- (when *token-check-happening*- (throw (ex-info "Token check is being called recursively, there is a good chance some `defenterprise` is causing this"- {:pass-thru true})))- ;; important to not count these errors against the circuit breaker. These are not the types of errors we need- ;; to circuit break. (#65294)- (when-not (app-db/db-is-set-up?)- (throw (ex-info "Metabase DB is not yet set up"- {:cause :token-check/app-db-not-ready})))- (locking lock- (binding [*token-check-happening* true]- (try (dh/with-circuit-breaker breaker- (dh/with-timeout {:timeout-ms timeout-ms- :interrupt? true}- (-check-token token-checker token)))- (catch dev.failsafe.CircuitBreakerOpenException _e- (throw (ex-info (tru "Token validation is currently unavailable.")- {:cause :token-check/circuit-breaker})))- ;; a timed-out execution has no cause to unwrap — translate it ourselves- (catch dev.failsafe.TimeoutExceededException _e- (throw (ex-info (tru "Token validation timed out.") {})))- ;; other exceptions are wrapped by Diehard in a FailsafeException. Unwrap them before- ;; rethrowing.- (catch dev.failsafe.FailsafeException e- (throw (or (.getCause e) e)))))))- (-clear-cache! [_]- (-clear-cache! token-checker))- (-clear-local-cache! [_]- (-clear-local-cache! token-checker)))))-(defn- hash-token"SHA-256 hex hash of a token string. Used as cache table key so the raw token never appears in plaintext."^String [^String token]@@ -408,11 +345,6 @@^String [result](-> result canonicalize-for-hash json/encode (.getBytes "UTF-8") buddy-hash/sha256 codecs/bytes->hex))-(defn- read-cache-from-db- "Read a cached token status hash from the premium_features_token_cache table. Returns nil if not found."- [token-hash]- (premium-features.db/token-status-cache token-hash))-(defn- write-cache-to-db!"Upsert a token status hash into the premium_features_token_cache table.Uses update-first to avoid poisoning PostgreSQL transactions on duplicate key.@@ -429,11 +361,6 @@;; Another instance inserted first — update instead.(premium-features.db/update-token-status-cache! token-hash result-hash now)))))))-(defn- clear-db-cache!- "Delete all rows from the premium_features_token_cache table."- []- (premium-features.db/delete-token-status-cache!))-(defn- extract-locks"Project a `:meters` map to `{meter-keyword -> boolean}` of `:is-locked` values.Meters without an `:is-locked` field are dropped."@@ -451,222 +378,309 @@(catch Throwable t(log/warnf "Failed to mirror :locked-meters from token-check response: %s" (ex-message t))))))-#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}-(def ^:dynamic *testing-only-call-after-refresh*- "When non-nil, a zero-arg function called after async background refresh completes.- For testing only — do not use in production."- nil)+;;; +----------------------------------------------------------------------------------------------------------------++;;; | REFRESHER |+;;; +----------------------------------------------------------------------------------------------------------------+-(defn db-hash-aware-token-checker- "Wraps a token checker with a DB-hash-aware local cache.- The DB stores only a hash of the token status (never the actual features). The full token status- lives exclusively in the local in-memory cache. The DB hash is used to:- 1. Validate that the local cache is consistent with what other instances see- 2. Detect when another instance has refreshed and got different features (hash mismatch)-- - `soft-ttl`: local entries younger than this are fresh; older triggers async refresh- - `hard-ttl`: local entries older than this are expired; synchronous fetch required- - `local-cache`: optional atom for the local cache (default: fresh atom). Exposed for testing.- - Hash mismatch or cold start always triggers a synchronous fetch."- [token-checker {:keys [soft-ttl hard-ttl local-cache]}]- (let [local-cache (or local-cache (atom {}))- refresh-in-progress? (atom false)]- (letfn [(do-refresh! [token token-hash]- (let [result (-check-token token-checker token)]- ;; a non-authoritative return here is an internal bug: every legitimate inner path- ;; returns a canonical map or throws- (when-not (:canonical? result)- (throw (ex-info "Refusing to cache a non-authoritative token status"- {:status (:status result)})))- (let [result-hash (hash-token-status result)- now (t/instant)]- (write-cache-to-db! token-hash result-hash)- (update-locked-meters! result)- (swap! local-cache assoc token-hash {:result result- :result-hash result-hash- :updated-at now})- result)))]- (reify TokenChecker- (-check-token [_ token]- (if-not (app-db/db-is-set-up?)- (-check-token token-checker token)- (let [token-hash (hash-token token)- local-entry (get @local-cache token-hash)- db-row (read-cache-from-db token-hash)- now (t/instant)]- (if (and local-entry- db-row- (= (:result-hash local-entry) (:token_status_hash db-row)))- ;; Local cache matches DB hash — check TTLs- (let [age-start (:updated-at local-entry)]- (cond- ;; Fresh (< soft-ttl): return local value- (t/before? now (t/plus age-start soft-ttl))- (:result local-entry)-- ;; Stale (soft..hard): return local + async refresh- (t/before? now (t/plus age-start hard-ttl))- (do- (when (compare-and-set! refresh-in-progress? false true)- (future- (try- (do-refresh! token token-hash)- (catch Exception e- (log/errorf "Background premium features refresh failed: %s" (ex-message e)))- (finally- (reset! refresh-in-progress? false)- (when-let [after *testing-only-call-after-refresh*]- (after))))))- (:result local-entry))-- ;; Expired (> hard-ttl): synchronous refresh- :else- (do-refresh! token token-hash)))- ;; No local cache, no DB row, or hash mismatch: synchronous fetch- (do-refresh! token token-hash)))))- (-clear-cache! [_]- (reset! local-cache {})- (when (app-db/db-is-set-up?)- (clear-db-cache!))- (-clear-cache! token-checker))- (-clear-local-cache! [_]- ;; `local-cache` is validated against the DB hash on every check, so it self-corrects; dropping it would- ;; only force a synchronous MetaStore fetch.- (-clear-local-cache! token-checker))))))--(defn local-cached-token-checker- "Wraps a token checker with short-lived in-memory TTL memoization.- Avoids hitting the DB on every request. Passes through uncached until the app DB is set up, so the- app-db-not-ready failure is never cached past setup."- [token-checker {:keys [local-ttl]}]- (let [cached-check (memoize/ttl- (fn [token] (-check-token token-checker token))- :ttl/threshold (.toMillis ^java.time.Duration local-ttl))]- (reify TokenChecker- (-check-token [_ token]- (if (app-db/db-is-set-up?)- (cached-check token)- (-check-token token-checker token)))- (-clear-cache! [_]- (memoize/memo-clear! cached-check)- (-clear-cache! token-checker))- (-clear-local-cache! [_]- (memoize/memo-clear! cached-check)- (-clear-local-cache! token-checker)))))--(defn- error-catching-token-checker- [token-checker]- (reify TokenChecker- (-check-token [_ token]- (try (-check-token token-checker token)- (catch Exception e- (when-not (-> e ex-data :cause #{:token-check/circuit-breaker :token-check/app-db-not-ready})- (log/infof "Error checking token: %s" (ex-message e))- (analytics/inc! :metabase-token-check/attempt {:status :failure}))- {:valid false- :canonical? false- :status (tru "Unable to validate token")- :error-details (.getMessage e)})))- (-clear-cache! [_] (-clear-cache! token-checker))- (-clear-local-cache! [_] (-clear-local-cache! token-checker))))+(def ^:private grace-period+ "How long a status is honored after it was fetched."+ (t/hours 36))-#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}-(def ^:dynamic *customize-checker*- "Dynamic variable allowing for customized token checkers. In the app, we want all of these in place. Only in tests- should we construct ones without circuit breakers "- false)--(defn make-checker- "Make a token checker. Takes a base [[TokenChecker]] token checker and then arguments for the middleware-style- wrapping [[TokenChecker]] arguments.-- `db-hash-local-cache` is optional and only provided during tests."- [{:keys [base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl db-hash-local-cache]- :or {base store-and-airgap-token-checker}}]- (when-not *customize-checker*- (assert (and base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl)- "Must provide all arguments for token checker"))- (cond-> base- circuit-breaker- (circuit-breaker-token-checker {:circuit-breaker circuit-breaker- :timeout-ms timeout-ms})-- (and soft-ttl hard-ttl)- (db-hash-aware-token-checker {:soft-ttl soft-ttl- :hard-ttl hard-ttl- :local-cache db-hash-local-cache})-- ;; inside the memoize so failure maps are memoized for local-ttl (negative cache; core.memoize- ;; single-flights values but not thrown exceptions), except before app-DB setup, when the memo- ;; is skipped; outside db-hash so they never reach the durable cache- :always- (error-catching-token-checker)-- local-ttl- (local-cached-token-checker {:local-ttl local-ttl})))--(def token-checker- "The token checker. Combines http/airgapping validation, circuit breaking, DB-hash-aware caching, and error handling."- (make-checker {:base store-and-airgap-token-checker- ;; the window must fit 10 worst-case failures (10s timeout + 5s negative cache- ;; each ≈ 135s) or slow failures can never open the breaker- :circuit-breaker {:failure-threshold-ratio-in-period [10 10 (u/seconds->ms 180)]- :delay-ms (u/seconds->ms 30)- :success-threshold 1- :on-open (fn [_] (log/info "Engaging circuit breaker in token check"))- :on-half-open (fn [_] (log/info "In token check circuit breaker but attempting check"))- :on-close (fn [_] (log/info "Token Check restored"))}- :timeout-ms (u/seconds->ms 10)- :local-ttl (t/seconds 5)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)}))+;; `{:token-hash, :status, :status-hash, :fetched-at}`, or nil+(defonce ^:private entry (atom nil))++;; token hash -> promise of `{:status ...}` or `{:error ...}`+(defonce ^:private in-flight (atom {}))++(defonce ^:private refresher-executor (atom nil))++;; `{:token-hash, :message}` of the latest failed refresh, or nil+(defonce ^:private last-failure (atom nil))++(def ^:private retry-policy+ {:max-retries 2+ :initial-interval-millis 1000+ :multiplier 2.0+ :jitter-factor 0.1+ :max-interval-millis 10000})++(defonce ^:private breaker+ (dh.cb/circuit-breaker {:failure-threshold 3+ :delay-ms (u/seconds->ms 30)+ :on-open (fn [_] (log/info "Engaging circuit breaker in token check"))+ :on-close (fn [_] (log/info "Token Check restored"))}))++(def ^:private fetch-timeout-ms+ "Caps a fetch, retries included. A blocked socket read is not interrupted: it ends at the HTTP client's own timeouts."+ (u/seconds->ms 10))++(defn- fetch-status+ [token & {:keys [retry?] :or {retry? true}}]+ (try+ (dh/with-circuit-breaker breaker+ (dh/with-timeout {:timeout-ms fetch-timeout-ms+ :interrupt? true}+ (if retry?+ (retry/with-retry (assoc retry-policy+ :on-retry (fn [_ e] (log/infof "Retrying token check after: %s" (ex-message e))))+ (decode-token* token))+ (decode-token* token))))+ (catch CircuitBreakerOpenException _e+ (throw (ex-info (tru "Token validation is currently unavailable.") {})))+ (catch TimeoutExceededException _e+ (throw (ex-info (tru "Token validation timed out.") {})))+ ;; Failsafe wraps checked exceptions, such as a `ConnectException`+ (catch FailsafeException e+ (throw (or (ex-cause e) e)))))++(defn- install!+ [token status]+ ;; the token may have been replaced while its status was being fetched+ (when (= token (premium-features.settings/premium-embedding-token))+ (let [token-hash (hash-token token)+ status-hash (hash-token-status status)]+ (reset! entry {:token-hash token-hash+ :status status+ :status-hash status-hash+ :fetched-at (t/instant)})+ (reset! last-failure nil)+ (try+ (write-cache-to-db! token-hash status-hash)+ (catch Throwable e+ (log/warnf "Failed to record the token status hash: %s" (ex-message e))))+ (update-locked-meters! status)))+ status)++(defn- refresh!+ [fetch-opts]+ (when-let [token (premium-features.settings/premium-embedding-token)]+ (let [token-hash (hash-token token)+ mine (promise)+ [before] (swap-vals! in-flight (fn [m] (cond-> m (not (contains? m token-hash)) (assoc token-hash mine))))+ shared (get before token-hash)]+ (when-not shared+ (try+ (deliver mine {:status (install! token (fetch-status token fetch-opts))})+ (log/info "Refreshed the premium features token status.")+ (catch Throwable e+ (log/errorf "Error refreshing the premium features token status: %s" (ex-message e))+ (analytics/inc! :metabase-token-check/attempt {:status :failure})+ (reset! last-failure {:token-hash token-hash :message (ex-message e)})+ (deliver mine {:error e}))+ (finally+ (swap! in-flight dissoc token-hash))))+ (let [{:keys [status error]} @(or shared mine)]+ (if error+ (throw error)+ status)))))++(defn refresh-now!+ "Fetch the status of the configured token and hold it. Concurrent callers share one fetch. Returns the fresh status,+ nil when no token is configured, or throws when the fetch fails, in which case the held status is left as it was."+ []+ (refresh! {}))++(defn refresh-at-startup!+ "Make one attempt, without retries, to fetch and hold the status of the configured token. A failure is left for+ [[tick!]] to retry, not thrown."+ []+ (try+ (refresh! {:retry? false})+ nil+ (catch Throwable _e+ nil)))(defn clear-cache!- "Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."+ "Refresh the token status now; see [[refresh-now!]]. A failure is logged, not thrown, and the held status stands."+ []+ (try+ (refresh-now!)+ nil+ (catch Throwable _e+ nil)))++(defonce ^:private refresh-interval+ ;; jittered per instance so a fleet started together does not fetch together+ (t/minus (t/hours 12) (t/seconds (rand-int 3600))))++(defonce ^:private refresh-requested? (atom false))++(defn request-refresh!+ "Have the next [[tick!]] refresh the token status, for a caller that should not wait for the fetch itself."+ []+ (reset! refresh-requested? true)+ nil)++(defn- shared-status-differs?+ "Whether the shared hash row for the held status is missing or records a different status, as it does once another+ instance has refreshed."+ [{:keys [token-hash status-hash]}]+ (not= (:token_status_hash (premium-features.db/token-status-cache token-hash)) status-hash))++(defn- refresh-reason+ [token requested?]+ (let [{:keys [token-hash fetched-at] :as held} @entry]+ (cond+ (not= token-hash (hash-token token))+ "no status is held"++ requested?+ "a refresh was requested"++ (t/before? (t/plus fetched-at refresh-interval) (t/instant))+ "the held status is due for its periodic refresh"++ (shared-status-differs? held)+ "the shared hash row is missing or records a different status")))++(defn tick!+ "Refresh the token status if one is due. Never throws, so a scheduler keeps calling it. A requested refresh that+ fails stays requested."[]- (-clear-cache! token-checker))+ (let [[requested?] (reset-vals! refresh-requested? false)]+ (try+ (when-let [token (premium-features.settings/premium-embedding-token)]+ (when-let [reason (refresh-reason token requested?)]+ (log/infof "Refreshing the premium features token status: %s." reason)+ (refresh-now!)))+ nil+ (catch Throwable e+ (when requested?+ (reset! refresh-requested? true))+ (log/errorf "Premium features token tick failed: %s" (ex-message e))+ nil))))(defn clear-local-cache!- "Clear only this process's in-memory token memoization; see [[-clear-local-cache!]]. Unlike [[clear-cache!]] this- never touches the shared DB cache, so it is safe to trigger from an untrusted request."+ "Refresh the token status now if no status is held for the configured token, or if the shared hash row is missing+ or records a different status than the one held; otherwise do nothing. Before [[start-refresher!]], only request the refresh,+ since initialization makes its own. A failure is logged, not thrown."[]- (-clear-local-cache! token-checker))+ (try+ (when-let [token (premium-features.settings/premium-embedding-token)]+ (let [held @entry]+ (when (or (not= (:token-hash held) (hash-token token))+ (shared-status-differs? held))+ (if @refresher-executor+ (clear-cache!)+ (request-refresh!)))))+ nil+ (catch Throwable e+ (log/errorf "Error checking whether the premium features token status is current: %s" (ex-message e))+ nil)))++(def ^:private tick-interval-ms (u/minutes->ms 1))-(defn check-token- "Public entrypoint to the token checking."- ([token] (check-token token-checker token))- ([checker token]- (-check-token checker token)))+(defn- safe-tick!+ []+ (try+ (tick!)+ ;; `scheduleWithFixedDelay` stops rescheduling a task that throws+ (catch Throwable t+ (log/errorf "Unexpected error in the premium features token tick; ticking continues: %s" (ex-message t)))))++(defn start-refresher!+ "Start a daemon thread that calls [[tick!]] every `interval-ms`, a minute by default. Returns `true` if this call+ started it, `false` if one was already running."+ ([]+ (start-refresher! tick-interval-ms))+ ([interval-ms]+ (let [exec (Executors/newSingleThreadScheduledExecutor+ (reify ThreadFactory+ (newThread [_ r]+ (doto (Thread. r "premium-features-token-refresher")+ (.setDaemon true)))))]+ (if (compare-and-set! refresher-executor nil exec)+ (do (.scheduleWithFixedDelay exec ^Runnable safe-tick! (long interval-ms) (long interval-ms) TimeUnit/MILLISECONDS)+ true)+ (do (.shutdown exec)+ false)))))++(defn stop-refresher!+ "Stop the thread started by [[start-refresher!]]."+ []+ (when-let [^ScheduledExecutorService exec (first (reset-vals! refresher-executor nil))]+ (.shutdownNow exec)+ (when-not (.awaitTermination exec 5 TimeUnit/SECONDS)+ (log/warn "The premium features token refresher did not stop within 5s."))))++;;; +----------------------------------------------------------------------------------------------------------------++;;; | READER |+;;; +----------------------------------------------------------------------------------------------------------------+++;; hash of the last token [[tick-for-new-token!]] ticked for, so each new token gets one early tick+(defonce ^:private ticked-for-token-hash (atom nil))++(defn- tick-for-new-token!+ "Have the refresher thread tick now for a token configured since the last refresh, as when another instance stored+ it, instead of serving no features until the next scheduled tick. Does not wait for the tick. A token whose refresh+ has already failed is left to the scheduled ticks."+ [token-hash]+ (when-let [^ScheduledExecutorService exec @refresher-executor]+ (when (and (not= token-hash (:token-hash @last-failure))+ (not= token-hash (first (reset-vals! ticked-for-token-hash token-hash))))+ (try+ (.execute exec ^Runnable safe-tick!)+ (catch RejectedExecutionException _e+ nil)))))++(defn- current-status+ []+ (when-let [token (premium-features.settings/premium-embedding-token)]+ (let [{:keys [token-hash status fetched-at]} @entry+ configured-hash (hash-token token)]+ (if (not= token-hash configured-hash)+ (do (tick-for-new-token! configured-hash)+ nil)+ (when (t/before? (t/instant) (t/plus fetched-at grace-period))+ status)))))++(defn unvalidated-status+ "The status reported for a token whose status could not be validated, with `error-details` saying why."+ ([]+ (unvalidated-status (tru "No current token status is held. A refresh may be in progress or failing.")))+ ([error-details]+ {:valid false+ :canonical? false+ :status (tru "Unable to validate token")+ :error-details error-details}))++(defn- validated-status+ "The store's status for `token`, a candidate for the premium token. Throws unless the token is valid."+ [token]+ (when (mr/validate [:re AirgapToken] token)+ (assert-valid-airgap-user-count!))+ (when-not (or (mr/validate [:re RemoteCheckedToken] token)+ (mr/validate [:re AirgapToken] token))+ (throw (ex-info (tru "Token format is invalid.")+ {:status-code 400+ :error-details "Token should be 64 hexadecimal characters."+ :response/keys #{:error-details}})))+ (let [status (try+ (fetch-status token)+ (catch Throwable e+ (throw (ex-info (tru "Unable to validate token")+ {:error-details (ex-message e)+ :status-code 503+ :response/keys #{:error-details}}+ e))))]+ (when-not (:valid status)+ (throw (ex-info (:status status)+ {:error-details (:error-details status)+ :status-code 400+ :response/keys #{:error-details}})))+ (log/info "Token is valid.")+ status))(defn -set-premium-embedding-token!"Setter for the [[metabase.premium-features.settings/token-status]] setting."[new-value]- ;; validate the new value if we're not unsetting it(try- (when (seq new-value)- (when (mr/validate [:re AirgapToken] new-value)- (assert-valid-airgap-user-count!))- (when-not (or (mr/validate [:re RemoteCheckedToken] new-value)- (mr/validate [:re AirgapToken] new-value))- (throw (ex-info (tru "Token format is invalid.")- {:status-code 400- :error-details "Token should be 64 hexadecimal characters."- :response/keys #{:error-details}})))- ;; validate against the store, not a cached verdict — the token may be the same string with a- ;; renewed subscription behind it- (clear-cache!)- (let [decoded (check-token new-value)]- (when-not (:valid decoded)- (throw (ex-info (:status decoded)- {:error-details (:error-details decoded)- ;; If MetaStore told us the token is invalid, use 400. If some other error occurred, a 503 is- ;; probably more appropriate.- :status-code (if (:canonical? decoded)- 400- 503)- :response/keys #{:error-details}}))))- (log/info "Token is valid."))- (setting/set-value-of-type! :string :premium-embedding-token new-value)+ ;; validate the new value if we're not unsetting it+ (let [status (when (seq new-value)+ (validated-status new-value))]+ (setting/set-value-of-type! :string :premium-embedding-token new-value)+ (when status+ (install! new-value status)))((requiring-resolve 'metabase.dependencies.hooks/after-premium-token-update))(catch Throwable e(log/errorf "Error setting premium features token: %s" (ex-message e))@@ -680,58 +694,47 @@[](mr/validate AirgapToken (premium-features.settings/premium-embedding-token)))-(let [cached-logger (memoize/ttl- ^{::memoize/args-fn (fn [[token _e]] [token])}- (fn [_token e]- (log/errorf "Error validating token: %s" (ex-message e)))- ;; log every five minutes- :ttl/threshold (* 1000 60 5))]- #_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}- (mu/defn ^:dynamic *token-features* :- [:set ms/NonBlankString]- "Get the features associated with the system's premium features token."- []- (try- ;; an invalid token (e.g. expired) confers no features, even if the token-status response still- ;; lists them (EMB-2341)- (or (let [{:keys [valid features]} (some-> (premium-features.settings/premium-embedding-token)- (check-token))]- (when valid- (set features)))- #{})- (catch Throwable e- (when (:pass-thru (ex-data e))- (throw e))- (cached-logger (premium-features.settings/premium-embedding-token) e)- #{}))))+#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}+(mu/defn ^:dynamic *token-features* :- [:set ms/NonBlankString]+ "Get the features associated with the system's premium features token."+ []+ (try+ ;; an invalid token (e.g. expired) confers no features, even if the token-status response still+ ;; lists them (EMB-2341)+ (let [{:keys [valid features]} (current-status)]+ (if valid+ (set features)+ #{}))+ (catch Throwable e+ (log/errorf "Error reading premium features: %s" (ex-message e))+ #{})))(defn -token-status"Getter for the [[metabase.premium-features.settings/token-status]] setting."[]- (some-> (premium-features.settings/premium-embedding-token)- (check-token)))+ (when-let [token (premium-features.settings/premium-embedding-token)]+ (or (current-status)+ (let [{:keys [token-hash message]} @last-failure]+ (if (and message (= token-hash (hash-token token)))+ (unvalidated-status message)+ (unvalidated-status))))))(mu/defn plan-alias :- [:maybe :string]"Returns a string representing the instance's current plan, if included in the last token status request."[]- (some-> (premium-features.settings/premium-embedding-token)- (check-token)- :plan-alias))+ (:plan-alias (current-status)))(mu/defn quotas :- [:maybe [:sequential [:map]]]"Returns a vector of maps for each quota of the subscription."[](clear-cache!)- (some-> (premium-features.settings/premium-embedding-token)- (check-token)- :quotas))+ (:quotas (current-status)))(mu/defn meters :- [:maybe :map]"Returns a map of current metered usage for the subscription."[](clear-cache!)- (some-> (premium-features.settings/premium-embedding-token)- (check-token)- :meters))+ (:meters (current-status)))(defn has-any-features?"True if we have a valid premium features token with ANY features."@@ -750,11 +753,9 @@"Returns `true` if the token definitively has `feature`, `false` if it definitively does not, or `nil` if the tokenstatus is indeterminate (e.g., network failure, timeout). Returns `false` (not `nil`) when no token is configured."[feature]- (if-let [token (premium-features.settings/premium-embedding-token)]- (let [result (check-token token)]- (when (:canonical? result)- (boolean (and (:valid result)- (contains? (set (:features result)) (name feature))))))+ (if (premium-features.settings/premium-embedding-token)+ (when-let [{:keys [valid features]} (current-status)]+ (boolean (and valid (contains? (set features) (name feature)))))false))(defn ee-feature-errordiff --git a/src/metabase/server/middleware/premium_features_cache.clj b/src/metabase/server/middleware/premium_features_cache.cljindex ea8c747a2d5..8267c18a7a6 100644--- a/src/metabase/server/middleware/premium_features_cache.clj+++ b/src/metabase/server/middleware/premium_features_cache.clj@@ -1,14 +1,13 @@(ns metabase.server.middleware.premium-features-cache"Ring middleware to propagate premium-features cache invalidation across instances via a cookie.When one instance refreshes features (via POST /api/premium-features/token/refresh), it sets a cookie- with the current timestamp. Other instances see this cookie and drop their in-process token caches so- the next feature check re-validates against the shared DB cache, and only re-fetches from the MetaStore- if that shows the other instance really did get a different token status.+ with the current timestamp. Other instances see this cookie and compare the status they hold with the shared hash+ row in the app DB, refreshing their own token status from the MetaStore only when no status is held or the row+ is missing or differs. A cookie therefore costs at most an app DB read unless another instance really has a newer status.The cookie is client-controlled and arrives on unauthenticated requests, so it is a hint, never anauthority: it must parse as a timestamp, must not be in the future, must be newer than the last- invalidation this instance saw, and can only ever clear this instance's in-memory caches. It never deletes- the deployment's shared DB cache."+ invalidation this instance saw, and can only ever trigger a refresh of this instance's own token status."(:require[metabase.premium-features.core :as premium-features][metabase.util.log :as log]@@ -31,7 +30,7 @@(def ^:private last-known-invalidation"The newest invalidation this instance has seen or emitted, as an `Instant`. Any cookie at or before it is ignored,- so a given cookie value can trigger a local clear at most once."+ so a given cookie value can trigger a local refresh at most once."(atom nil))(defn- parse-cookie-timestamp@@ -59,12 +58,12 @@(not= old new)))(defn- check-and-invalidate-features-cache- "If the request cookie carries a valid timestamp newer than the last invalidation this instance knows about, drop- this instance's in-process token caches. Never touches the shared DB cache."+ "If the request cookie carries a valid timestamp newer than the last invalidation this instance knows about, have+ this instance refresh its token status if it is out of date."[request](when-let [cookie-timestamp (parse-cookie-timestamp (get-in request [:cookies cookie-name :value]))](when (advance-last-known-invalidation! cookie-timestamp)- (log/info "Premium features cookie indicates local cache is out of date. Clearing...")+ (log/info "Premium features cookie indicates the local token status may be out of date. Checking...")(premium-features/clear-local-cache!))))(defn- maybe-set-premium-features-cookie@@ -82,10 +81,10 @@response))(defn wrap-premium-features-cache-check- "Middleware that checks if the local premium features cache needs to be invalidated based on a cookie.+ "Middleware that checks if the local premium features token status needs a refresh based on a cookie.When premium features are refreshed on one instance, the API sets a cookie with the current timestamp.- This middleware checks that cookie on subsequent requests and clears the local cache if needed,+ This middleware checks that cookie on subsequent requests and refreshes the local status if it is out of date,ensuring consistency in multi-instance deployments."[handler](fn [request respond raise]diff --git a/test/metabase/premium_features/rest/api_test.clj b/test/metabase/premium_features/rest/api_test.cljindex 489310afc96..37b6b848224 100644--- a/test/metabase/premium_features/rest/api_test.clj+++ b/test/metabase/premium_features/rest/api_test.clj@@ -4,8 +4,11 @@[clj-http.cookies :as cookies][clojure.test :refer :all][metabase.premium-features.core :as premium-features]+ [metabase.premium-features.test-util :as premium-features.tu][metabase.premium-features.token-check :as token-check]- [metabase.test :as mt])+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [metabase.util.retry :as retry])(:import(clojure.lang ExceptionInfo)))@@ -33,23 +36,72 @@(deftest post-token-refresh-test(testing "POST /api/premium-features/token/refresh"- (testing "clears cache and returns fresh token status"- (let [cleared? (atom false)]- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)- premium-features/premium-embedding-token (constantly nil)- token-check/clear-cache! (fn [] (reset! cleared? true))]+ (testing "refreshes and returns the fresh token status"+ (let [refreshed? (atom false)]+ (mt/with-dynamic-fn-redefs [premium-features/premium-embedding-token (constantly nil)+ token-check/refresh-now! (fn []+ (reset! refreshed? true)+ fake-token-status)](is (=? (dissoc fake-token-status :trial)(mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))- (is (true? @cleared?))))))+ (is (true? @refreshed?))))))(testing "requires superusers"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :post 403 "premium-features/token/refresh"))))(testing "returns 404 if no token is set"- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly nil)+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly nil)premium-features/premium-embedding-token (constantly nil)](is (= "Not found."(mt/user-http-request :crowberto :post 404 "premium-features/token/refresh"))))))+(deftest post-token-refresh-returns-fresh-status-test+ (testing "POST /api/premium-features/token/refresh fetches from the store and returns what it said"+ (let [entry (var-get #'token-check/entry)+ saved @entry+ status (atom {:valid true :status "ok" :features ["sandboxes"]})]+ (try+ (mt/with-temp-env-var-value! [mb-premium-embedding-token nil]+ (mt/with-temporary-raw-setting-values [premium-embedding-token (premium-features.tu/random-token)]+ (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _] {:status 200 :body (json/encode @status)})]+ (is (=? {:valid true :features ["sandboxes"]}+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))+ (reset! status {:valid true :status "ok" :features ["sandboxes" "audit-app"]})+ (is (=? {:valid true :features ["sandboxes" "audit-app"]}+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))+ (is (= #{"sandboxes" "audit-app"} (token-check/*token-features*))))))+ (finally+ (reset! entry saved))))))++(deftest post-token-refresh-reports-a-failed-check-test+ (testing "POST /api/premium-features/token/refresh reports a failed store call, while feature checks keep the held status"+ (let [entry (var-get #'token-check/entry)+ saved @entry+ fail? (atom false)]+ (try+ (mt/with-temp-env-var-value! [mb-premium-embedding-token nil]+ (mt/with-temporary-raw-setting-values [premium-embedding-token (premium-features.tu/random-token)]+ (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]+ (if @fail?+ (throw (ex-info "network failure!" {}))+ {:status 200+ :body (json/encode {:valid true+ :status "ok"+ :features ["sandboxes"]})}))]+ (is (=? {:valid true :features ["sandboxes"]}+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))+ (reset! fail? true)+ (binding [retry/*test-time-config-hook* #(assoc % :initial-interval-millis 1 :max-interval-millis 2)]+ (is (= {:valid false+ :canonical? false+ :status "Unable to validate token"+ :error-details "network failure!"}+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh"))))+ (is (premium-features/has-feature? :sandboxes))+ (is (=? {:valid true :features ["sandboxes"]} (premium-features/token-status))))))+ (finally+ (.close ^dev.failsafe.CircuitBreaker @#'token-check/breaker)+ (reset! entry saved))))))+(deftest post-token-refresh-invalidates-llm-proxy-cache-test(testing "POST /api/premium-features/token/refresh invalidates the AI service token cache when explicitly configured"(let [token "SOME_RANDOM_TOKEN"]@@ -59,7 +111,7 @@(mt/with-temporary-setting-values [llm-proxy-base-url "https://proxy.example.com/llm/"ai-service-base-url "https://ai-service.example.com/"](let [request* (atom nil)]- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly fake-token-status)http/post (fn [url request-options](reset! request* [url request-options]){:status 200})]@@ -68,7 +120,7 @@(is (= (str "https://ai-service.example.com/v1/invalidate-token-cache/" token) url))(is (false? (:throw-exceptions opts))))))))))))(testing "POST /api/premium-features/token/refresh does not invalidate the AI service cache when it is not configured"- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly fake-token-status)premium-features/premium-embedding-token (constantly "proxy-token")http/post (fn [& _](throw (ex-info "should not be called" {})))]@@ -79,7 +131,7 @@(testing "POST /api/premium-features/token/refresh sets the premium-features-last-updated cookie";; user-real-request hits a real Jetty server; handler thread doesn't inherit *local-redefs*.#_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}- (with-redefs [premium-features/token-status (constantly fake-token-status)+ (with-redefs [token-check/refresh-now! (constantly fake-token-status)premium-features/premium-embedding-token (constantly nil)](let [cs (cookies/cookie-store)](mt/user-real-request :crowberto :post 200 "premium-features/token/refresh"@@ -100,7 +152,7 @@(mt/with-premium-features #{:metabase-ai-managed}(mt/with-temp-env-var-value! [mb-premium-embedding-token nil](mt/with-temporary-raw-setting-values [premium-embedding-token token]- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly fake-token-status)http/post (fn [url opts] (reset! request* [url opts]) {:status 200})](testing "a stored base URL gets no more than llm-allowed-networks allows"(mt/with-temporary-setting-values [ai-service-base-url "http://10.0.0.1/"]diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.cljindex 8c53a64330e..3e62b95b41c 100644--- a/test/metabase/premium_features/token_check_test.clj+++ b/test/metabase/premium_features/token_check_test.clj@@ -4,21 +4,20 @@[clj-http.core :as http.core][clojure.test :refer :all][java-time.api :as t]- [mb.hawk.parallel][metabase.app-db.connection :as mdb.connection][metabase.app-db.core :as mdb][metabase.premium-features.core :as premium-features][metabase.premium-features.db :as premium-features.db]- [metabase.premium-features.task.clear-token-cache]+ [metabase.premium-features.settings :as premium-features.settings][metabase.premium-features.test-util :as tu][metabase.premium-features.token-check :as token-check]- [metabase.startup.core :as startup][metabase.test :as mt][metabase.test.fixtures :as fixtures][metabase.util :as u][metabase.util.api-error :as api-error][metabase.util.json :as json][metabase.util.malli.registry :as mr]+ [metabase.util.retry :as retry][methodical.core :as methodical][toucan2.core :as t2][toucan2.pipeline :as t2.pipeline]))@@ -27,480 +26,610 @@(use-fixtures :once (fixtures/initialize :db))-(defn- fresh-checker- "A token checker with production TTLs and no circuit breaker.- The global checker's breaker is shared across the JVM, and one tripped by an earlier test would fail every check here."- []- (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/seconds 5)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)})))--(deftest log-tests- (let [token (tu/random-token)- print-token (apply str (concat (take 4 token) "..." (take-last 4 token)))]- (testing "Do not log the token (#18249)"- (mt/with-log-messages-for-level [messages :info]- (token-check/check-token (fresh-checker) token)- (let [logs (mapv :message (messages))]- (is (every? (complement #(re-find (re-pattern token) %)) logs))- (is (= 1 (count (filter #(re-find (re-pattern print-token) %) logs)))))))))--(deftest fetch-token-caches-invalid-responses- (testing "For 4XX responses, the result is cached"- (let [call-count (atom 0)- token (tu/random-token)]- (binding [http/request (fn [& _]- (swap! call-count inc)- {:status 400 :body "{\"valid\": false, \"status\": \"fake\"}"})]- (let [checker (fresh-checker)]- (dotimes [_ 10] (token-check/check-token checker token)))- (is (= 1 @call-count))))))--(deftest fetch-token-does-not-cache-exceptions- (let [call-count (atom 0)- token (tu/random-token)- response (atom :error)- ;; no circuit breaker (carries state between blocks) and no local-ttl (it negative-caches- ;; failures) — this pins that the DURABLE tiers never store them- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:soft-ttl (t/minutes 1)- :hard-ttl (t/minutes 2)}))]+(defn- do-with-clean-refresher+ [thunk]+ (let [entry @#'token-check/entry+ breaker @#'token-check/breaker+ saved @entry]+ (reset! entry nil)+ (reset! @#'token-check/refresh-requested? false)+ (reset! @#'token-check/last-failure nil)+ (reset! @#'token-check/ticked-for-token-hash nil)+ (.close ^dev.failsafe.CircuitBreaker breaker)+ (try+ (binding [retry/*test-time-config-hook* #(assoc % :initial-interval-millis 1 :max-interval-millis 2)]+ (thunk))+ (finally+ (.close ^dev.failsafe.CircuitBreaker breaker)+ (reset! entry saved)))))++(use-fixtures :each do-with-clean-refresher)++(defn- do-with-store+ [respond f]+ (let [calls (atom 0)](mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- (case @response- :error (throw (ex-info "kaboom" {:ka :boom}))- :500 {:status 500}- :400-empty {:status 400}- :400-html {:status 400 :body "<html>Bad Gateway</html>"}- :200-empty {:status 200}))]- (testing "For timeouts, 5XX errors, etc. we don't cache the result"- (dotimes [_ 5] (token-check/check-token checker token))- (is (= 5 @call-count)))- (testing "Does not cache 500 responses"- (reset! call-count 0)- (reset! response :500)- (dotimes [_ 5] (token-check/check-token checker token))- (is (= 5 @call-count)))- (testing "A 4XX with no body is not a real answer from the store — treated as transient, not cached"- (reset! call-count 0)- (reset! response :400-empty)- (dotimes [_ 5] (token-check/check-token checker token))- (is (= 5 @call-count))- (is (= {:valid false- :canonical? false- :status "Unable to validate token"- :error-details "Token validation provided no response."}- (token-check/check-token checker token))))- (testing "An unparseable body (e.g. a proxy's error page) is transient, not cached, and surfaces a legible error"- (reset! call-count 0)- (reset! response :400-html)- (dotimes [_ 5] (token-check/check-token checker token))- (is (= 5 @call-count))- (is (=? {:canonical? false- :error-details "Token validation provided no response."}- (token-check/check-token checker token))))- (testing "A 2XX with no body is transient, not cached, and surfaces a legible error"- (reset! call-count 0)- (reset! response :200-empty)- (dotimes [_ 5] (token-check/check-token checker token))- (is (= 5 @call-count))- (is (=? {:canonical? false- :error-details "Token validation provided no response."}- (token-check/check-token checker token)))))))--(deftest not-found-test- (mt/with-log-level :fatal- (is (=? {:valid false, :status "Token does not exist."}- (token-check/check-token (fresh-checker) (tu/random-token))))))--(deftest expired-token-status-is-authoritative-test- (testing "a 403 with a body is a real answer from the store: stored faithfully, marked canonical (EMB-2341)"- (let [token (tu/random-token)- status {:valid false+ (swap! calls inc)+ (let [response (respond)]+ (if (contains? response :valid)+ {:status 200 :body (json/encode response)}+ response)))]+ (f calls))))++(defmacro ^:private with-store+ {:style/indent 1, :clj-kondo/lint-as 'clojure.core/let}+ [[calls-binding respond] & body]+ `(do-with-store ~respond (fn [~calls-binding] ~@body)))++(defn- hash-row [token]+ (t2/select-one :model/PremiumFeaturesCache :token_hash (#'token-check/hash-token token)))++(deftest successful-refresh-populates-entry-and-hash-row-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [calls (constantly {:valid true :status "ok" :features ["sandboxes" "audit-app"]})]+ (testing "before any refresh there are no features"+ (is (= #{} (token-check/*token-features*))))+ (is (=? {:valid true :features ["sandboxes" "audit-app"]}+ (token-check/refresh-now!)))+ (is (= 1 @calls))+ (is (= #{"sandboxes" "audit-app"} (token-check/*token-features*)))+ (is (premium-features/has-feature? :sandboxes))+ (is (=? {:valid true :status "ok"} (premium-features/token-status)))+ (is (some? (hash-row token)))))))++(defn- age-entry!+ [age]+ (swap! @#'token-check/entry update :fetched-at t/minus age))++(def ^:private good-status {:valid true :status "ok" :features ["sandboxes"]})++(deftest failed-refresh-leaves-held-status-alone-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!))+ (let [row (hash-row token)]+ (with-store [calls #(throw (ex-info "network failure!" {}))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"network failure!" (token-check/refresh-now!)))+ (is (= 3 @calls) "the refresh retries before giving up")+ (is (= #{"sandboxes"} (token-check/*token-features*)) "the previously held features keep serving")+ (is (= row (hash-row token)) "the hash row is untouched"))))))++(deftest failed-first-refresh-holds-nothing-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [_ #(throw (ex-info "network failure!" {}))]+ (is (thrown? clojure.lang.ExceptionInfo (token-check/refresh-now!)))+ (is (= #{} (token-check/*token-features*)))+ (is (nil? (hash-row token)))+ (is (=? {:valid false :canonical? false :status "Unable to validate token"}+ (premium-features/token-status)))))))++(deftest transient-responses-are-never-held-test+ (doseq [[description response] {"a 5xx" {:status 500}+ "a 2xx with no body" {:status 200}+ "a bare 403" {:status 403}+ "a bare 404" {:status 404}+ "a proxy's HTML error page" {:status 400 :body "<html>Bad Gateway</html>"}+ "an empty object" {:status 200 :body "{}"}+ "a non-boolean valid" {:status 200 :body "{\"valid\":\"yes\",\"status\":\"ok\"}"}+ "a blank status" {:status 200 :body "{\"valid\":false,\"status\":\"\"}"}+ "features that aren't a list" {:status 200 :body "{\"valid\":true,\"status\":\"ok\",\"features\":\"sso\"}"}+ "a JSON value, not an object" {:status 200 :body "[1,2,3]"}}]+ (testing (format "%s carries no verdict from the store: the refresh fails and the held status stands" description)+ (reset! @#'token-check/entry nil)+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!))+ (with-store [_ (constantly response)]+ (is (thrown? clojure.lang.ExceptionInfo (token-check/refresh-now!)))+ (is (= #{"sandboxes"} (token-check/*token-features*))))))))++(deftest rejection-with-a-body-is-authoritative-test+ (testing "a 4xx carrying a status is the store's answer: it replaces the held status, and confers no features (EMB-2341)"+ (let [expired {:valid false:status "Token is expired."- :features ["sso" "snippet-collections" "database-routing"]+ :features ["sso" "snippet-collections"]:plan-alias "pro-self-hosted"- :trial false- :valid-thru "2026-08-25T12:00:00Z"}- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/seconds 5)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _] {:status 403 :body (json/encode status)})]- (is (= (assoc status :canonical? true)- (token-check/check-token checker token))))- (finally- (token-check/-clear-cache! checker))))))--(deftest invalid-token-in-2xx-is-authoritative-test- (testing "an invalid verdict in a 2xx is a real answer: cached, and features gated the same as a 4xx"- (let [token (tu/random-token)- call-count (atom 0)- body (json/encode {:valid false :status "Token is expired." :features ["sso"]})- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/seconds 5)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- {:status 200 :body body})]- (is (= {:valid false :status "Token is expired." :features ["sso"] :canonical? true}- (token-check/check-token checker token)))- (token-check/check-token checker token)- (is (= 1 @call-count) "authoritative answers are cached"))- (finally- (token-check/-clear-cache! checker))))))--(deftest hung-request-times-out-test- (testing "a request that never returns times out with a legible error and is not cached"- (let [token (tu/random-token)- call-count (atom 0)- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker- {:base (reify token-check/TokenChecker- (-check-token [_ _]- (swap! call-count inc)- (Thread/sleep 5000))- (-clear-cache! [_])- (-clear-local-cache! [_]))- :circuit-breaker {:failure-threshold-ratio-in-period [10 10 1000]- :delay-ms 50- :success-threshold 1}- :timeout-ms 100- :local-ttl (t/millis 1)- :soft-ttl (t/minutes 1)- :hard-ttl (t/minutes 2)}))]- (try- (is (= {:valid false- :canonical? false- :status "Unable to validate token"- :error-details "Token validation timed out."}- (token-check/check-token checker token)))- (Thread/sleep 5) ;; expire the local memoize layer- (token-check/check-token checker token)- (is (= 2 @call-count) "timeouts are not cached")- (finally- (token-check/-clear-cache! checker))))))--(deftest transient-failures-are-negative-cached-test- ;; timing rule: asserting behavior WITHIN a TTL window needs a TTL generous enough that test- ;; overhead can't overrun it (10s); sleeping PAST a tight TTL is deterministic (sleep ≥ duration)- (testing "a transient failure is memoized for local-ttl — an outage costs one attempt per TTL, not one per feature check"- (let [token (tu/random-token)- call-count (atom 0)- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/seconds 10)- :soft-ttl (t/minutes 1)- :hard-ttl (t/minutes 2)}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- (throw (ex-info "network failure!" {})))]- (dotimes [_ 5] (token-check/check-token checker token))- (is (= 1 @call-count) "failures within the TTL are served from the negative cache")- (is (=? {:valid false :canonical? false :error-details "network failure!"}- (token-check/check-token checker token))))- (finally- (token-check/-clear-cache! checker)))))- (testing "the negative cache expires with the TTL — failures are retried"- (let [token (tu/random-token)- call-count (atom 0)- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/millis 50)- :soft-ttl (t/minutes 1)- :hard-ttl (t/minutes 2)}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- (throw (ex-info "network failure!" {})))]- (token-check/check-token checker token)- (is (= 1 @call-count))+ :valid-thru "2026-08-25T12:00:00Z"}]+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!))+ (with-store [_ (constantly {:status 403 :body (json/encode expired)})]+ (is (= (assoc expired :canonical? true) (token-check/refresh-now!)))+ (is (= #{} (token-check/*token-features*)))+ (is (false? (premium-features/has-feature? :sso)))+ (is (false? (premium-features/canonically-has-feature? :sso)))+ (is (= "pro-self-hosted" (premium-features/plan-alias)))+ (is (= (assoc expired :canonical? true) (premium-features/token-status))))))))++(deftest invalid-status-in-2xx-confers-no-features-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ (constantly {:valid false :status "Token is expired." :features ["sso"]})]+ (token-check/refresh-now!)+ (is (= #{} (token-check/*token-features*)))+ (is (false? (premium-features/canonically-has-feature? :sso))))))++(deftest feature-check-reads-no-app-db-test+ (doseq [held? [false true]]+ (testing (if held? "with a held status" "with no held status")+ (reset! @#'token-check/entry nil)+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (when held?+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!)))+ ;; the first read may run the settings cache's own once-a-minute staleness check+ (premium-features/has-feature? :sandboxes)+ (with-store [calls (constantly good-status)]+ (t2/with-call-count [call-count]+ (dotimes [_ 10]+ (is (= held? (premium-features/has-feature? :sandboxes)))+ (premium-features/token-status))+ (is (zero? (call-count))))+ (is (zero? @calls)))))))++(deftest feature-check-does-not-wait-for-a-refresh-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!))+ (let [fetching (promise)+ release (promise)]+ (with-store [_ (fn []+ (deliver fetching true)+ @release+ {:valid true :status "ok" :features ["audit-app"]})]+ (let [refresh (future (token-check/refresh-now!))]+ (is (true? (deref fetching 5000 :timeout)))+ (is (= #{"sandboxes"} (token-check/*token-features*)) "the held status serves while the refresh is in flight")+ (deliver release true)+ (is (=? {:features ["audit-app"]} (deref refresh 5000 :timeout)))+ (is (= #{"audit-app"} (token-check/*token-features*))))))))++(deftest concurrent-refreshes-share-one-fetch-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (let [fetching (promise)+ release (promise)]+ (with-store [calls (fn []+ (deliver fetching true)+ @release+ good-status)]+ (let [first-refresh (future (token-check/refresh-now!))+ _ (deref fetching 5000 :timeout)+ others (mapv (fn [_] (future (token-check/refresh-now!))) (range 7))](Thread/sleep 100)- (token-check/check-token checker token)- (is (= 2 @call-count)))- (finally- (token-check/-clear-cache! checker)))))- (testing "concurrent misses share a single attempt (core.memoize single-flights values, and failures are values here)"- (let [token (tu/random-token)- call-count (atom 0)- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/seconds 10)- :soft-ttl (t/minutes 1)- :hard-ttl (t/minutes 2)}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- (Thread/sleep 100)- (throw (ex-info "network failure!" {})))]- (let [checks (mapv (fn [_] (future (token-check/check-token checker token))) (range 8))]- (is (every? #(false? (:valid %)) (mapv deref checks)))- (is (= 1 @call-count) "8 concurrent callers, one network attempt")))- (finally- (token-check/-clear-cache! checker))))))--(deftest bare-rejection-is-transient-test- (doseq [http-status [403 404]]- (testing (format "a bare %d (no body) carries no verdict from the store: a proxy may have stripped it, so it is transient, not cached"- http-status)- (let [token (tu/random-token)- call-count (atom 0)- ;; no local-ttl: it negative-caches failures, and this pins that the durable tiers never store them- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:soft-ttl (t/minutes 1)- :hard-ttl (t/minutes 2)}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- {:status http-status})]- (dotimes [_ 5] (token-check/check-token checker token))- (is (= 5 @call-count) "not cached: every check goes back to the store")- (is (=? {:valid false- :canonical? false- :error-details "Token validation provided no response."}- (token-check/check-token checker token))))- (finally- (token-check/-clear-cache! checker)))))))--(deftest malformed-status-body-is-transient-test- (doseq [[description body] {"an empty object" "{}"- "an unrelated object" "{\"foo\":1}"- "a non-boolean valid" "{\"valid\":\"yes\",\"status\":\"ok\"}"- "a blank status" "{\"valid\":false,\"status\":\"\"}"- "features that aren't a list" "{\"valid\":true,\"status\":\"ok\",\"features\":\"sso\"}"- "a JSON value, not an object" "[1,2,3]"}]- (testing (format "%s decodes as JSON but is not a token status: transient, never stamped canonical or cached" description)- (let [token (tu/random-token)- call-count (atom 0)- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:soft-ttl (t/minutes 1)- :hard-ttl (t/minutes 2)}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- {:status 200 :body body})]- (dotimes [_ 3] (token-check/check-token checker token))- (is (= 3 @call-count) "not cached")- (is (=? {:valid false- :canonical? false- :error-details "Token validation provided no response."}- (token-check/check-token checker token))))- (finally- (token-check/-clear-cache! checker)))))))--(deftest do-refresh-refuses-non-canonical-test- (testing "the caching layer refuses to hold on to a result not marked :canonical?"- (let [token (tu/random-token)- call-count (atom 0)- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker- {:base (reify token-check/TokenChecker- (-check-token [_ _]- (swap! call-count inc)- {:valid true :status "fake" :features ["sso"]})- (-clear-cache! [_])- (-clear-local-cache! [_]))- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)}))]+ (deliver release true)+ (is (every? #(= (assoc good-status :canonical? true) (deref % 5000 :timeout))+ (cons first-refresh others)))+ (is (= 1 @calls)))))))++(deftest grace-period-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ (testing "a status younger than 36h is honored"+ (age-entry! (t/hours 35))+ (is (= #{"sandboxes"} (token-check/*token-features*)))+ (is (true? (premium-features/canonically-has-feature? :sandboxes))))+ (testing "an older one is not"+ (age-entry! (t/hours 2))+ (is (= #{} (token-check/*token-features*)))+ (is (nil? (premium-features/canonically-has-feature? :sandboxes)))+ (is (=? {:valid false :canonical? false :status "Unable to validate token"}+ (premium-features/token-status))))+ (is (= 1 @calls) "reading never fetches"))))++(deftest no-entry-before-app-db-setup-test+ (testing "with no held status and no app DB, a feature check yields nothing and does not throw"+ (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly false)]+ (is (= #{} (token-check/*token-features*)))+ (is (false? (premium-features/has-feature? :sandboxes))))))++(deftest status-for-another-token-is-never-served-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!)))+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (is (= #{} (token-check/*token-features*)))+ (is (=? {:valid false :canonical? false} (premium-features/token-status)))))++(deftest airgap-refresh-makes-no-http-call-test+ (let [airgap-status {:valid true :status "Token is valid" :features ["sandboxes"] :max-users 10}]+ (mt/with-dynamic-fn-redefs [token-check/decode-airgap-token (constantly airgap-status)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token "airgap_fake.token"]+ (with-store [calls #(throw (ex-info "no HTTP call expected" {}))]+ (is (= (assoc airgap-status :canonical? true) (token-check/refresh-now!)))+ (is (= #{"sandboxes"} (token-check/*token-features*)))+ (is (zero? @calls)))))))++(deftest refresh-mirrors-locked-meters-test+ (mt/with-temporary-setting-values [locked-meters {}]+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (let [response (atom nil)]+ (with-store [_ #(deref response)]+ (testing "a status with :meters writes through"+ (reset! response {:valid true :status "ok" :meters {:transform-basic-runs {:is-locked true}+ :transform-advanced-runs {:is-locked false}}})+ (token-check/refresh-now!)+ (is (= {:transform-basic-runs true :transform-advanced-runs false} (premium-features/locked-meters))))+ (testing "a status without :meters leaves the setting alone"+ (reset! response {:valid true :status "ok"})+ (token-check/refresh-now!)+ (is (= {:transform-basic-runs true :transform-advanced-runs false} (premium-features/locked-meters))))+ (testing "an empty :meters unlocks everything"+ (reset! response {:valid true :status "ok" :meters {}})+ (token-check/refresh-now!)+ (is (= {} (premium-features/locked-meters))))+ (testing "a failed refresh leaves the setting alone (a network blip must never unlock)"+ (reset! response {:valid true :status "ok" :meters {:transform-basic-runs {:is-locked true}}})+ (token-check/refresh-now!)+ (reset! response {:status 503})+ (is (thrown? clojure.lang.ExceptionInfo (token-check/refresh-now!)))+ (is (= {:transform-basic-runs true} (premium-features/locked-meters)))))))))++(deftest breaker-stops-fetching-after-consecutive-failures-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls #(throw (ex-info "network failure!" {}))]+ (dotimes [_ 3]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"network failure!" (token-check/refresh-now!))))+ (reset! calls 0)+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Token validation is currently unavailable"+ (token-check/refresh-now!)))+ (is (zero? @calls)))))++(deftest fetch-is-capped-by-a-timeout-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-redefs [token-check/fetch-timeout-ms 100]+ (with-store [_ #(do (Thread/sleep 5000) good-status)]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Token validation timed out"+ (token-check/refresh-now!)))))))++(deftest checked-fetch-exceptions-are-unwrapped-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ #(throw (java.net.ConnectException. "Connection refused"))]+ (is (thrown-with-msg? java.net.ConnectException #"Connection refused" (token-check/refresh-now!))))))++(deftest token-status-reports-the-latest-failure-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [_ #(throw (java.net.ConnectException. "Connection refused"))]+ (premium-features/clear-cache!)+ (is (=? {:valid false :canonical? false :error-details "Connection refused"}+ (premium-features/token-status))))+ (testing "a successful refresh clears it"+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!))+ (is (=? {:valid true :status "ok"} (premium-features/token-status)))))+ (testing "another token's failure is not reported"+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [_ #(throw (java.net.ConnectException. "Connection refused"))]+ (premium-features/clear-cache!)))+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (is (=? {:valid false :canonical? false :error-details #"No current token status is held.*"}+ (premium-features/token-status)))))))++(deftest clear-cache-refreshes-and-never-throws-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ (constantly good-status)]+ (is (nil? (premium-features/clear-cache!)))+ (is (= #{"sandboxes"} (token-check/*token-features*))))+ (with-store [_ #(throw (ex-info "network failure!" {}))]+ (is (nil? (premium-features/clear-cache!)))+ (is (= #{"sandboxes"} (token-check/*token-features*))))))++(defn- do-with-refresher-started!+ [thunk]+ ;; an hour between ticks, so none runs during the test+ (premium-features/start-refresher! (u/hours->ms 1))+ (try+ (thunk)+ (finally+ (premium-features/stop-refresher!))))++(deftest tick-with-no-entry-held-fetches-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (token-check/tick!)+ (is (= 1 @calls))+ (is (premium-features/has-feature? :sandboxes)))))++(deftest tick-with-a-fresh-matching-entry-does-nothing-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ (let [row (hash-row token)]+ (token-check/tick!)+ (is (= 1 @calls))+ (is (= row (hash-row token)))+ (is (=? {:valid true :status "ok"} (premium-features/token-status))))))))++(def ^:private audit-app-status {:valid true :status "ok" :features ["audit-app"]})++(defn- change-hash-row!+ [token]+ (t2/update! :model/PremiumFeaturesCache :token_hash (#'token-check/hash-token token)+ {:token_status_hash (#'token-check/hash-token-status audit-app-status)}))++(deftest clear-local-cache-with-a-matching-hash-row-does-not-fetch-test+ (do-with-refresher-started!+ (fn []+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ (dotimes [_ 3]+ (premium-features/clear-local-cache!))+ (is (= 1 @calls))+ (is (not @@#'token-check/refresh-requested?))+ (is (premium-features/has-feature? :sandboxes)))))))++(deftest clear-local-cache-with-a-differing-hash-row-fetches-test+ (do-with-refresher-started!+ (fn []+ (let [token (tu/random-token)+ response (atom good-status)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [calls #(deref response)]+ (token-check/refresh-now!)+ (reset! response audit-app-status)+ (change-hash-row! token)+ (premium-features/clear-local-cache!)+ (is (= 2 @calls))+ (is (premium-features/has-feature? :audit-app))+ (is (not (premium-features/has-feature? :sandboxes)))+ (is (=? {:valid true :features ["audit-app"]} (premium-features/token-status)))+ (testing "the hash row now matches the held status, so the next hint does nothing"+ (premium-features/clear-local-cache!)+ (is (= 2 @calls)))))))))++(deftest clear-local-cache-with-no-held-status-fetches-test+ (do-with-refresher-started!+ (fn []+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (premium-features/clear-local-cache!)+ (is (= 1 @calls))+ (is (premium-features/has-feature? :sandboxes)))))))++(deftest read-for-a-newly-configured-token-ticks-early-test+ (do-with-refresher-started!+ (fn []+ (let [calls (atom 0)]+ ;; the tick runs on the refresher thread, which does not see `with-dynamic-fn-redefs`+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ (with-redefs [token-check/http-fetch (fn [& _]+ (swap! calls inc)+ {:status 200 :body (json/encode good-status)})]+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (is (not (premium-features/has-feature? :sandboxes))+ "the read itself does not wait for the fetch")+ (is (true? (u/poll {:thunk #(premium-features/has-feature? :sandboxes)+ :done? true?+ :timeout-ms 5000+ :interval-ms 10})))+ (is (= 1 @calls))+ (testing "a token whose early tick already ran gets no second one"+ (reset! @#'token-check/entry nil)+ (premium-features/has-feature? :sandboxes)+ (Thread/sleep 200)+ (is (= 1 @calls)))))+ (testing "a token whose refresh already failed waits for the scheduled tick"+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ ;; a tick, if one ran, would be on the refresher thread+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ (with-redefs [token-check/http-fetch (fn [& _]+ (swap! calls inc)+ (throw (java.net.ConnectException. "Connection refused")))]+ (premium-features/refresh-at-startup!)+ (reset! calls 0)+ (premium-features/has-feature? :sandboxes)+ (Thread/sleep 200)+ (is (zero? @calls)))))))))++(deftest clear-local-cache-whose-fetch-fails-does-not-throw-test+ (do-with-refresher-started!+ (fn []+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls #(throw (ex-info "network failure!" {}))]+ (is (nil? (premium-features/clear-local-cache!)))+ (is (pos? @calls)))))))++(deftest clear-local-cache-before-the-refresher-starts-only-requests-a-refresh-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [calls (constantly good-status)]+ (testing "with nothing held"+ (premium-features/clear-local-cache!)+ (is (zero? @calls) "a hint that arrives during initialization does not fetch")+ (is @@#'token-check/refresh-requested?)+ (token-check/tick!)+ (is (= 1 @calls) "the first tick makes the refresh the hint asked for"))+ (testing "with a differing hash row"+ (change-hash-row! token)+ (premium-features/clear-local-cache!)+ (is (= 1 @calls))+ (is @@#'token-check/refresh-requested?)+ (token-check/tick!)+ (is (= 2 @calls))+ (is (premium-features/has-feature? :sandboxes)))))))++(deftest tick-after-another-instance-changed-the-hash-fetches-test+ (let [token (tu/random-token)+ response (atom good-status)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [calls #(deref response)]+ (token-check/refresh-now!)+ (reset! response audit-app-status)+ (change-hash-row! token)+ (token-check/tick!)+ (is (= 2 @calls))+ (is (premium-features/has-feature? :audit-app))+ (is (not (premium-features/has-feature? :sandboxes)))+ (testing "the next tick sees the hash it now holds and does nothing"+ (token-check/tick!)+ (is (= 2 @calls)))))))++(deftest tick-with-a-missing-hash-row-fetches-and-writes-it-back-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ (t2/delete! :model/PremiumFeaturesCache :token_hash (#'token-check/hash-token token))+ (token-check/tick!)+ (is (= 2 @calls) "a missing row triggers one fetch")+ (is (= (:status-hash @@#'token-check/entry) (:token_status_hash (hash-row token))))+ (is (premium-features/has-feature? :sandboxes))+ (testing "the row is back, so the next tick does nothing"+ (token-check/tick!)+ (is (= 2 @calls)))))))++(deftest tick-after-the-refresh-interval-fetches-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ (testing "an entry younger than 11h, the shortest jittered interval, is not refreshed"+ (age-entry! (t/minus (t/hours 11) (t/minutes 1)))+ (token-check/tick!)+ (is (= 1 @calls)))+ (testing "an entry older than 12h, the longest jittered interval, is"+ (age-entry! (t/minutes 62))+ (token-check/tick!)+ (is (= 2 @calls))+ (is (premium-features/has-feature? :sandboxes)))+ (testing "the refreshed entry is fresh again"+ (token-check/tick!)+ (is (= 2 @calls))))))++(deftest tick-after-a-requested-refresh-fetches-once-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ (token-check/request-refresh!)+ (is (= 1 @calls) "requesting a refresh does not fetch")+ (token-check/tick!)+ (is (= 2 @calls))+ (testing "the request is cleared, so the next tick does nothing"+ (token-check/tick!)+ (is (= 2 @calls))))))++(deftest tick-whose-fetch-fails-leaves-the-entry-alone-test+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!))+ (let [held @@#'token-check/entry+ row (hash-row token)]+ (with-store [calls #(throw (ex-info "network failure!" {}))]+ (token-check/request-refresh!)+ (is (nil? (token-check/tick!)))+ (is (pos? @calls))+ (is (= held @@#'token-check/entry))+ (is (= row (hash-row token)))+ (is (premium-features/has-feature? :sandboxes)))+ (testing "the request stands, so the next tick tries again"+ (with-store [calls (constantly good-status)]+ (token-check/tick!)+ (is (= 1 @calls))+ (token-check/tick!)+ (is (= 1 @calls))))))))++(deftest tick-whose-hash-read-fails-does-not-throw-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ (mt/with-dynamic-fn-redefs [premium-features.db/token-status-cache (fn [_] (throw (ex-info "app DB down" {})))]+ (is (nil? (token-check/tick!))))+ (is (= 1 @calls))+ (is (premium-features/has-feature? :sandboxes)))))++(deftest tick-reads-the-app-db-at-most-once-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (token-check/refresh-now!)+ ;; the first read may run the settings cache's own once-a-minute staleness check+ (token-check/tick!)+ (t2/with-call-count [call-count]+ (token-check/tick!)+ (is (= 1 (call-count)) "a tick with nothing due reads the hash row and nothing else"))+ (t2/with-call-count [call-count]+ (is (premium-features/has-feature? :sandboxes))+ (is (zero? (call-count)) "a feature check reads nothing"))+ (is (= 1 @calls)))))++(defn- fetch-log-count+ [messages]+ (count (filter #(re-find #"Checking with the MetaStore" (:message %)) messages)))++(deftest startup-refresh-on-a-healthy-store-fetches-once-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls (constantly good-status)]+ (mt/with-log-messages-for-level [messages :info]+ (is (nil? (premium-features/refresh-at-startup!)))+ (is (= 1 (fetch-log-count (messages)))))+ (is (= 1 @calls))+ (is (premium-features/has-feature? :sandboxes)))))++(deftest startup-refresh-on-an-unreachable-store-tries-once-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [calls #(throw (java.net.SocketTimeoutException. "Read timed out"))]+ (is (nil? (premium-features/refresh-at-startup!)) "the failure does not escape")+ (is (= 1 @calls) "the attempt is not retried")+ (is (not (premium-features/has-feature? :sandboxes)))+ (is (=? {:valid false :canonical? false} (premium-features/token-status))))+ (testing "a later tick picks up features once the store returns"+ (with-store [calls (constantly good-status)]+ (token-check/tick!)+ (is (= 1 @calls))+ (is (premium-features/has-feature? :sandboxes))))))++(defn- refresher-threads []+ (filter #(= "premium-features-token-refresher" (.getName ^Thread %)) (keys (Thread/getAllStackTraces))))++(deftest refresher-ticks-until-stopped-test+ (let [ticks (atom 0)]+ ;; the executor's thread does not inherit dynamic bindings+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}+ (with-redefs [token-check/tick! (fn []+ (when (odd? (swap! ticks inc))+ (throw (ex-info "tick failure!" {}))))](try- (dotimes [_ 3] (token-check/check-token checker token))- (is (= 3 @call-count) "non-canonical results are not cached")- (is (=? {:valid false- :canonical? false- :error-details "Refusing to cache a non-authoritative token status"}- (token-check/check-token checker token)))+ (is (true? (premium-features/start-refresher! 1)))+ (is (false? (premium-features/start-refresher! 1)) "a second start leaves the running refresher alone")+ (is (u/poll {:thunk #(deref ticks) :done? #(>= % 4) :interval-ms 1})+ "it keeps ticking after a tick throws")+ (is (= 1 (count (refresher-threads))))(finally- (token-check/-clear-cache! checker))))))--(deftest token-features-honors-valid-test- (testing "*token-features* returns no features when the token status is not valid, even if features are present"- (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]- (mt/with-dynamic-fn-redefs [token-check/check-token (fn [_token] {:valid false- :canonical? true- :status "Token is expired."- :features ["sso"]})]- (is (= #{} (token-check/*token-features*)))- (is (false? (token-check/has-feature? :sso)))- (is (false? (token-check/canonically-has-feature? :sso)))))))--(deftest fetch-token-does-not-call-db-when-cached- (testing "No DB calls are made when checking token status if the status is in local cache"- (let [token (tu/random-token)- checker (fresh-checker)- _ (token-check/check-token checker token)- ;; The local cache has a 5s TTL, so repeated checks within that window should not hit the DB.- call-counts (repeatedly 3 (fn []- (t2/with-call-count [call-count]- (token-check/check-token checker token)- (call-count))))]- ;; At least some of these should be zero (served from local in-memory cache)- (is (some zero? call-counts)))))--(deftest token-checker-test- (let [token (tu/random-token)- behavior (atom :success)- call-count (atom 0)- good-response {:valid true- :status "fake"- :features ["feature1" "feature2"]- :canonical? true}- token-response (fn [_token]- (swap! call-count inc)- (case @behavior- :success good-response- :error (throw (ex-info "network issues!" {:ka :boom}))))- checker (token-check/make-checker- {:base (reify token-check/TokenChecker- (-check-token [_ token]- (token-response token))- (-clear-cache! [_])- (-clear-local-cache! [_]))- :circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]- :delay-ms 50- :success-threshold 1}- :timeout-ms 100- :local-ttl (t/millis 50)- :soft-ttl (t/millis 200)- :hard-ttl (t/seconds 1)})]- (testing "when no information is present, hits the underlying"- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count)))- (testing "hitting it again in the same timeperiod does not hit the underlying checker"- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count)))- (testing "While it errors, we have circuit breaking"- (reset! behavior :error)- (Thread/sleep 200)- (dotimes [_ 1000] (token-check/-check-token checker token))- (is (< @call-count 50)))- (testing "When connectivity is restored we get successful token checks"- (reset! behavior :success)- (Thread/sleep 100) ;; wait for circuit breaker to open back up (also outlasts the 50ms negative cache)- (is (= good-response (token-check/-check-token checker token)))))- (testing "App db not setup yet does not invoke the circuit breaker (#65294), and the refusal is not cached past setup"- (let [token (tu/random-token)- call-count (atom 0)- good-response {:valid true- :status "fake"- :features ["feature1" "feature2"]- :canonical? true}- checker (token-check/make-checker- {:base (reify token-check/TokenChecker- (-check-token [_ _token]- (swap! call-count inc)- good-response)- (-clear-cache! [_])- (-clear-local-cache! [_]))- :circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]- :delay-ms 5000- :success-threshold 1}- :timeout-ms 100- :local-ttl (t/seconds 10)- :soft-ttl (t/millis 200)- :hard-ttl (t/seconds 1)})]- (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly false)]- (is (= {:valid false- :canonical? false- :status "Unable to validate token"- :error-details "Metabase DB is not yet set up"}- (token-check/-check-token checker token)))- (dotimes [_ 50] (token-check/-check-token checker token))- (is (= {:valid false- :canonical? false- :status "Unable to validate token"- :error-details "Metabase DB is not yet set up"}- (token-check/-check-token checker token))))- (testing "When the db-is-set-up? we are not blocked by the circuit breaker, and the first check reaches the store"+ (premium-features/stop-refresher!)))+ (is (u/poll {:thunk refresher-threads :done? empty? :interval-ms 1}) "stopping leaves no thread behind")+ (testing "it can be started again after a stop"(try- (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly true)]- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count)))+ (is (true? (premium-features/start-refresher! 1)))(finally- (token-check/-clear-cache! checker)))))))--(defn- age-cache-entry!- "Rewrite the `updated_at` timestamp for `token` in the cache table to be `age-ms` milliseconds in the past.- Also ages the local cache entry if a `local-cache` atom is provided."- ([token age-ms]- (age-cache-entry! token age-ms nil))- ([token age-ms local-cache]- (let [token-hash (#'token-check/hash-token token)- new-ts (t/minus (t/offset-date-time) (t/millis age-ms))]- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash {:updated_at new-ts})- (when local-cache- (swap! local-cache update token-hash assoc :updated-at (t/minus (t/instant) (t/millis age-ms)))))))--(deftest e2e-test- (testing "full stack: HTTP fetch → DB-hash-aware cache → error handling"- (let [token (tu/random-token)- call-count (atom 0)- good-body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"- good-resp {:valid true, :status "fake", :features ["fake" "features"], :canonical? true}- local-cache (atom {})- ;; no circuit breaker — it carries state between runs and causes flakes- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/millis 50)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)- :db-hash-local-cache local-cache}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- {:status 200 :body good-body})]- (is (= good-resp (token-check/check-token checker token)))- (is (= 1 @call-count)))- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- (throw (ex-info "network failure!" {})))]- (testing "Doesn't hit the network inside of cache duration"- ;; Wait for local-cached-token-checker TTL to expire, but DB-hash-aware cache is still fresh- (Thread/sleep 60)- (is (= good-resp (token-check/check-token checker token)))- (is (= 1 @call-count)))- (testing "Stale cache (past soft TTL, within hard TTL) still serves cached result"- (age-cache-entry! token (+ (u/hours->ms 12) 1000) local-cache)- (Thread/sleep 60) ;; expire local-cached-token-checker- (let [refresh-done (promise)]- (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]- (is (= good-resp (token-check/check-token checker token))))- ;; async refresh was attempted (and failed), but stale value was returned- (is (true? (deref refresh-done 5000 :timeout))))- (is (= 2 @call-count)))- (testing "Expired cache (past hard TTL) surfaces the error"- (age-cache-entry! token (+ (u/hours->ms 36) 1000) local-cache)- (Thread/sleep 60) ;; expire local-cached-token-checker- (is (= {:valid false- :canonical? false- :status "Unable to validate token"- :error-details "network failure!"}- (token-check/check-token checker token)))))- (finally- (token-check/-clear-cache! checker))))))--(deftest license-server-fields-we-do-not-read-test- (testing "a token status response carrying fields we don't read still validates the token"- (let [token (tu/random-token)- body {:valid true- :status "ok"- :features ["sso-jwt"]- :plan-alias "pro-self-hosted"- :new-field "added by the license server"- :store-users [{:email "[email protected]" :id 42 :first-name "Owner" :last-name "Person"}]- :quotas [{:hosting-feature "metabase-ai-tokens"- :soft-limit 100- :usage 5- :locked false- :updated-at "2026-09-01T00:00:00Z"- :quota-type "monthly"}]}- checker (fresh-checker)]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _] {:status 200 :body (json/encode body)})]- (is (= (assoc body :canonical? true)- (token-check/check-token checker token))))- (finally- (token-check/-clear-cache! checker))))))+ (premium-features/stop-refresher!)))))))++(deftest quotas-and-meters-reflect-a-fresh-fetch-test+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (let [usage (atom 1)]+ (with-store [calls (fn [] {:valid true+ :status "ok"+ :quotas [{:hosting-feature "metabase-ai-tokens" :usage @usage}]+ :meters {:metabase-ai-tokens {:meter-value @usage}}})]+ (is (=? [{:usage 1}] (premium-features/quotas)))+ (swap! usage inc)+ (is (=? {:metabase-ai-tokens {:meter-value 2}} (premium-features/meters)))+ (is (= 2 @calls))))))++(deftest log-test+ (let [token (tu/random-token)+ print-token (apply str (concat (take 4 token) "..." (take-last 4 token)))]+ (testing "Do not log the token (#18249)"+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [_ (constantly good-status)]+ (mt/with-log-messages-for-level [messages :info]+ (token-check/refresh-now!)+ (let [logs (mapv :message (messages))]+ (is (every? (complement #(re-find (re-pattern token) %)) logs))+ (is (= 1 (count (filter #(re-find (re-pattern print-token) %) logs)))))))))))(deftest ^:parallel extract-locks-test(testing "empty :meters map yields empty result"@@ -521,72 +650,7 @@(#'token-check/extract-locks {:transform-basic-runs {:is-locked true}:metabase-ai-tokens {:is-locked false}})))))-(deftest do-refresh-writes-locked-meters-test- (testing "do-refresh! mirrors :meters → :locked-meters setting on every successful refresh"- (mt/with-temporary-setting-values [locked-meters {}]- (let [token (tu/random-token)- response (atom {:valid true :status "ok" :canonical? true})- local-cache (atom {})- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/millis 50)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)- :db-hash-local-cache local-cache}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch- (fn [& _] {:status 200 :body (json/encode @response)})]- (testing "successful response with :meters writes through"- (reset! response {:valid true :status "ok"- :meters {:transform-basic-runs {:is-locked true}- :transform-advanced-runs {:is-locked false}}})- (token-check/check-token checker token)- (is (= {:transform-basic-runs true- :transform-advanced-runs false}- (premium-features/locked-meters))))- (testing "successful response WITHOUT :meters leaves the setting untouched"- (reset! response {:valid true :status "ok"})- (token-check/-clear-cache! checker)- (reset! local-cache {})- (token-check/check-token checker token)- (is (= {:transform-basic-runs true- :transform-advanced-runs false}- (premium-features/locked-meters))- "Setting should retain previous value when response omits :meters"))- (testing "successful response with empty :meters {} writes empty map (legitimate unlock)"- (reset! response {:valid true :status "ok" :meters {}})- (token-check/-clear-cache! checker)- (reset! local-cache {})- (token-check/check-token checker token)- (is (= {} (premium-features/locked-meters)))))- (finally- (token-check/-clear-cache! checker)))))))--(deftest do-refresh-failure-leaves-locked-meters-untouched-test- (testing "Outage / 5xx / circuit breaker open → :locked-meters survives unchanged.- (Critical correctness property: a network blip MUST NOT accidentally unlock.)"- (mt/with-temporary-setting-values [locked-meters {:transform-basic-runs true}]- (let [token (tu/random-token)- local-cache (atom {})- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:local-ttl (t/millis 50)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)- :db-hash-local-cache local-cache}))]- (try- (mt/with-dynamic-fn-redefs [token-check/http-fetch- (fn [& _] (throw (ex-info "network failure!" {})))]- (token-check/check-token checker token)- (is (= {:transform-basic-runs true}- (premium-features/locked-meters))- "Failed refresh must not touch :locked-meters"))- (finally- (token-check/-clear-cache! checker)))))))--(deftest token-status-setting-test- (testing "If a `premium-embedding-token` has been set, the `token-status` setting should return the response- from the store.metabase.com endpoint for that token."- (is (= {:valid false, :status "Token does not exist.", :canonical? true}- (token-check/check-token (fresh-checker) (tu/random-token)))))+(deftest token-status-without-token-test(testing "If premium-embedding-token is nil, the token-status setting should also be nil."(mt/with-temporary-setting-values [premium-embedding-token nil](is (nil? (premium-features/token-status))))))@@ -720,17 +784,12 @@(let [;; This is a fake airgap token format (starts with "airgap_")airgap-token "airgap_eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ90faketoken"request-made (atom false)]- (mt/with-dynamic-fn-redefs [token-check/check-token- (constantly {:valid true- :status "fake"- :features ["test" "fixture"]- :trial false})]- (mt/with-temporary-raw-setting-values [premium-embedding-token airgap-token]- (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts]- (reset! request-made true)- {:status 200 :body "{}"})]- (token-check/send-metering-events!)- (is (false? @request-made) "No request should be made for airgap tokens")))))))+ (mt/with-temporary-raw-setting-values [premium-embedding-token airgap-token]+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts]+ (reset! request-made true)+ {:status 200 :body "{}"})]+ (token-check/send-metering-events!)+ (is (false? @request-made) "No request should be made for airgap tokens"))))))(deftest metering-stats-test(testing "metering-stats returns expected keys"@@ -775,380 +834,81 @@:let [plan (-> (t2/query (into [(str "EXPLAIN " sql)] params)) first :plan)]](is (re-find #"IDX_QUERY_EXECUTION_STARTED_AT: STARTED_AT" plan)))))))-;;; ------------------------------------------------ db-hash-aware-token-checker --------------------------------------------------(defn- make-db-hash-aware-checker- "Helper: wraps a base token-response fn with DB-hash-aware caching. Returns the checker."- [token-response-fn {:keys [local-ttl soft-ttl hard-ttl]- :or {local-ttl (t/millis 50)}}]- (binding [token-check/*customize-checker* true]- (token-check/make-checker- {:base (reify token-check/TokenChecker- (-check-token [_ token] (token-response-fn token))- (-clear-cache! [_])- (-clear-local-cache! [_]))- :local-ttl local-ttl- :soft-ttl soft-ttl- :hard-ttl hard-ttl})))--(deftest clear-local-cache-test- (testing "clear-local-cache! drops in-process memoization but leaves the shared DB row alone"- (let [token (tu/random-token)- token-hash (#'token-check/hash-token token)- call-count (atom 0)- deletes (atom 0)- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- local-cache (atom {})- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker- {:base (reify token-check/TokenChecker- (-check-token [_ _token]- (swap! call-count inc)- good-response)- (-clear-cache! [_])- (-clear-local-cache! [_]))- ;; long enough that nothing expires during the test- :local-ttl (t/hours 1)- :soft-ttl (t/hours 12)- :hard-ttl (t/hours 36)- :db-hash-local-cache local-cache}))]- (try- (is (= good-response (token-check/check-token checker token)))- (is (= 1 @call-count))- (mt/with-dynamic-fn-redefs [premium-features.db/delete-token-status-cache! (fn [] (swap! deletes inc))]- (token-check/-clear-local-cache! checker))- (is (zero? @deletes) "the shared DB row is never deleted")- (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))- (is (contains? @local-cache token-hash) "the DB-hash-validated local entry survives")- (testing "the next check re-validates against the DB hash and, since it matches, skips the MetaStore"- (is (= good-response (token-check/check-token checker token)))- (is (= 1 @call-count)))- (testing "but a refresh done by another instance (DB hash changed) is picked up"- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash {:token_status_hash "refreshed-elsewhere"})- (testing "not while the in-process memo still holds"- (is (= good-response (token-check/check-token checker token)))- (is (= 1 @call-count)))- (token-check/-clear-local-cache! checker)- (is (= good-response (token-check/check-token checker token)))- (is (= 2 @call-count)))- (finally- (token-check/-clear-cache! checker))))))--(deftest db-hash-aware-token-checker-fresh-hit-test- (testing "Fresh cache entry (< soft-ttl) is returned without hitting the underlying checker"- (let [call-count (atom 0)- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- checker (make-db-hash-aware-checker- (fn [_token]- (swap! call-count inc)- good-response)- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})- token (tu/random-token)]- (try- ;; First call: cache miss → delegate- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count))- ;; Second call: fresh cache hit → no delegate- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count))- (finally- (token-check/-clear-cache! checker))))))--(deftest db-hash-aware-token-checker-stale-async-refresh-test- (testing "Stale entry (soft..hard) returns cached value and triggers async refresh"- (let [call-count (atom 0)- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- updated-response {:valid true :status "OK" :features ["sandboxes" "audit-app"] :canonical? true}- response-atom (atom good-response)- refresh-done (promise)- checker (make-db-hash-aware-checker- (fn [_token]- (swap! call-count inc)- @response-atom)- ;; soft=1ms so it's immediately stale, hard=10s so not expired- {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/seconds 10)})- token (tu/random-token)]- (try- ;; First call: cache miss → delegate- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count))- ;; Let the entry become stale (both local and DB)- (Thread/sleep 10)- ;; Switch to updated response for the async refresh- (reset! response-atom updated-response)- ;; Second call: stale → returns old cached value, kicks off async refresh- (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]- (is (= good-response (token-check/-check-token checker token)))- ;; bounded deref — unbounded hangs the whole run if the refresh never fires- (is (true? (deref refresh-done 5000 :timeout))))- ;; Expire local cache- (Thread/sleep 10)- ;; Third call: should now see the refreshed value- (is (= updated-response (token-check/-check-token checker token)))- (finally- (token-check/-clear-cache! checker))))))--(deftest db-hash-aware-token-checker-expired-sync-test- (testing "Expired entry (> hard-ttl) delegates synchronously"- (let [call-count (atom 0)- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- checker (make-db-hash-aware-checker- (fn [_token]- (swap! call-count inc)- good-response)- ;; Both TTLs = 1ms so entry expires immediately- {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})- token (tu/random-token)]- (try- ;; First call: miss- (token-check/-check-token checker token)- (is (= 1 @call-count))- (Thread/sleep 5)- ;; Second call: expired → synchronous delegate- (token-check/-check-token checker token)- (is (= 2 @call-count))- (finally- (token-check/-clear-cache! checker))))))--(deftest db-hash-aware-token-checker-multi-token-test- (testing "Different tokens are cached independently"- (let [call-count (atom 0)- checker (make-db-hash-aware-checker- (fn [token]- (swap! call-count inc)- {:valid true :status "OK" :features [(str "feat-" token)] :canonical? true})- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})- token-a (tu/random-token)- token-b (tu/random-token)]- (try- (let [result-a (token-check/-check-token checker token-a)]- (is (= [(str "feat-" token-a)] (:features result-a))))- (let [result-b (token-check/-check-token checker token-b)]- (is (= [(str "feat-" token-b)] (:features result-b))))- (is (= 2 @call-count))- ;; Both cached now- (token-check/-check-token checker token-a)- (token-check/-check-token checker token-b)- (is (= 2 @call-count))- (finally- (token-check/-clear-cache! checker))))))--(deftest db-hash-aware-token-checker-clear-cache-test- (testing "clear-cache! removes both local and DB cache"- (let [checker (make-db-hash-aware-checker- (fn [_token]- {:valid true :status "OK" :features ["sandboxes"] :canonical? true})- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})- token (tu/random-token)]- (token-check/-check-token checker token)- (let [token-hash (#'token-check/hash-token token)]- (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))- (token-check/-clear-cache! checker)- (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))))))--(deftest db-hash-aware-token-checker-invalid-cached-test- (testing "Invalid token responses are also cached in table to avoid hammering MetaStore"- (let [call-count (atom 0)- checker (make-db-hash-aware-checker- (fn [_token]- (swap! call-count inc)- {:valid false :status "Token does not exist." :canonical? true})- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})- token (tu/random-token)]- (try- (is (= {:valid false :status "Token does not exist." :canonical? true}- (token-check/-check-token checker token)))- (is (= 1 @call-count))- ;; Second call should use cache- (is (= {:valid false :status "Token does not exist." :canonical? true}- (token-check/-check-token checker token)))- (is (= 1 @call-count))- (finally- (token-check/-clear-cache! checker))))))--(deftest db-hash-aware-token-checker-cross-instance-sync-test- (testing "When instance A refreshes and gets new features, instance B detects the hash mismatch and re-fetches"- (let [call-count-a (atom 0)- call-count-b (atom 0)- old-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- new-response {:valid true :status "OK" :features ["sandboxes" "audit-app"] :canonical? true}- response-a (atom old-response)- response-b (atom old-response)- checker-a (make-db-hash-aware-checker- (fn [_token]- (swap! call-count-a inc)- @response-a)- {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})- checker-b (make-db-hash-aware-checker- (fn [_token]- (swap! call-count-b inc)- @response-b)- {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})- token (tu/random-token)]- (try- ;; Both instances populate their caches with old-response- (is (= old-response (token-check/-check-token checker-a token)))- (is (= old-response (token-check/-check-token checker-b token)))- (is (= 1 @call-count-a))- (is (= 1 @call-count-b))- ;; Instance A gets new features and refreshes (expired TTL forces sync refresh)- (reset! response-a new-response)- (Thread/sleep 5) ;; expire checker-a's local + DB TTLs- (is (= new-response (token-check/-check-token checker-a token)))- (is (= 2 @call-count-a))- ;; Instance B has a long TTL, but the DB hash now differs from its local hash- ;; So it should detect the mismatch and do a synchronous re-fetch- (reset! response-b new-response)- (Thread/sleep 5) ;; expire local-cached-token-checker's 1ms TTL- (is (= new-response (token-check/-check-token checker-b token)))- (is (= 2 @call-count-b))- (finally- (token-check/-clear-cache! checker-a)- (token-check/-clear-cache! checker-b))))))--(deftest db-hash-aware-token-checker-exception-propagation-test- (testing "When delegate throws on expired/missing, exception propagates through error-catching"- (let [checker (make-db-hash-aware-checker- (fn [_token]- (throw (ex-info "MetaStore unreachable" {})))- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})- token (tu/random-token)]- ;; error-catching wraps it, so we get the error-details response- (is (= {:valid false- :canonical? false- :status "Unable to validate token"- :error-details "MetaStore unreachable"}- (token-check/-check-token checker token))))))--(deftest db-hash-aware-token-checker-tamper-resistance-test- (testing "Tampering with the DB hash triggers a re-fetch from MetaStore, not a cache hit"- (let [call-count (atom 0)- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- checker (make-db-hash-aware-checker- (fn [_token]- (swap! call-count inc)- good-response)- {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})- token (tu/random-token)- token-hash (#'token-check/hash-token token)]- (try- ;; Populate the cache- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count))- ;; Tamper with the DB hash — simulate someone modifying the row- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash- {:token_status_hash "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"})- ;; Expire local-cached-token-checker- (Thread/sleep 5)- ;; Next check should detect hash mismatch and re-fetch from MetaStore- (is (= good-response (token-check/-check-token checker token)))- (is (= 2 @call-count))- (finally- (token-check/-clear-cache! checker))))))--(deftest startup-clears-token-cache-test- (testing "The startup hook clears the token cache table so the first check after boot hits MetaStore"- (let [call-count (atom 0)- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- checker (make-db-hash-aware-checker- (fn [_token]- (swap! call-count inc)- good-response)- {:soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})- token (tu/random-token)]- (try- ;; Populate the cache- (token-check/-check-token checker token)- (is (= 1 @call-count))- (let [token-hash (#'token-check/hash-token token)]- (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))- ;; Run the startup hook (clears the global token-checker's cache, including the table)- (startup/def-startup-logic! :metabase.premium-features.task.clear-token-cache/clear-token-cache)- ;; Table should be cleared- (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash))))- (finally- (token-check/-clear-cache! checker))))))--(deftest db-hash-aware-token-checker-db-not-set-up-test- (testing "When DB is not set up, delegates directly to inner checker without touching the DB"- (let [call-count (atom 0)- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}- checker (make-db-hash-aware-checker- (fn [_token]- (swap! call-count inc)- good-response)- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})- token (tu/random-token)- bomb (fn [& _] (throw (ex-info "DB should not be touched" {})))]- (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly false)- token-check/read-cache-from-db bomb- token-check/write-cache-to-db! bomb- token-check/clear-db-cache! bomb]- (is (= good-response (token-check/-check-token checker token)))- (is (= 1 @call-count) "inner checker was called exactly once")- ;; clear-cache! should also skip DB without error- (token-check/-clear-cache! checker)))))+(deftest license-server-fields-we-do-not-read-test+ (testing "a token status response carrying fields we don't read still validates the token"+ (let [body {:valid true+ :status "ok"+ :features ["sso-jwt"]+ :plan-alias "pro-self-hosted"+ :new-field "added by the license server"+ :store-users [{:email "[email protected]" :id 42 :first-name "Owner" :last-name "Person"}]+ :quotas [{:hosting-feature "metabase-ai-tokens"+ :soft-limit 100+ :usage 5+ :locked false+ :updated-at "2026-09-01T00:00:00Z"+ :quota-type "monthly"}]}]+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]+ (with-store [_ (constantly body)]+ (is (= (assoc body :canonical? true) (token-check/refresh-now!)))+ (is (= #{"sso-jwt"} (token-check/*token-features*))))))));;; ------------------------------------------------ -set-premium-embedding-token! ------------------------------------------------(deftest set-premium-embedding-token-canonical-invalid-test(testing "When MetaStore says the token is invalid (canonical), throws 400 with MetaStore's message"- (let [token (tu/random-token)]- (mt/with-dynamic-fn-redefs [token-check/check-token (constantly {:valid false- :status "Token expired"- :error-details "Expired 2024-01-01"- :canonical? true})]- (let [e (try (token-check/-set-premium-embedding-token! token)- nil- (catch Exception e e))]- (is (some? e))- (is (= "Token expired" (ex-message e)))- (is (= 400 (:status-code (ex-data e))))- (is (= "Expired 2024-01-01" (:error-details (ex-data e)))))))))--(deftest set-premium-embedding-token-non-canonical-failure-test- (testing "When token validation fails for non-canonical reasons (network etc), throws 503"- (let [token (tu/random-token)]- (mt/with-dynamic-fn-redefs [token-check/check-token (constantly {:valid false- :status "Unable to validate token"- :error-details "Connection refused"})]- (let [e (try (token-check/-set-premium-embedding-token! token)- nil- (catch Exception e e))]- (is (some? e))- (is (= "Unable to validate token" (ex-message e)))- (is (= 503 (:status-code (ex-data e))))- (is (= "Connection refused" (:error-details (ex-data e)))))))))+ (let [old-token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token old-token]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!))+ (with-store [_ (constantly {:status 403 :body (json/encode {:valid false+ :status "Token expired"+ :error-details "Expired 2024-01-01"})})]+ (let [e (try (token-check/-set-premium-embedding-token! (tu/random-token))+ nil+ (catch Exception e e))]+ (is (= "Token expired" (ex-message e)))+ (is (= 400 (:status-code (ex-data e))))+ (is (= "Expired 2024-01-01" (:error-details (ex-data e)))))+ (testing "the rejected token is not stored, and the old token's features keep serving"+ (is (= old-token (premium-features/premium-embedding-token)))+ (is (= #{"sandboxes"} (token-check/*token-features*)))))))))++(deftest set-premium-embedding-token-failure-test+ (testing "When the store cannot be reached, throws 503"+ (with-store [_ #(throw (ex-info "Connection refused" {}))]+ (let [e (try (token-check/-set-premium-embedding-token! (tu/random-token))+ nil+ (catch Exception e e))]+ (is (= "Unable to validate token" (ex-message e)))+ (is (= 503 (:status-code (ex-data e))))+ (is (= "Connection refused" (:error-details (ex-data e))))))))(deftest set-premium-embedding-token-valid-test- (testing "When token is valid, no exception is thrown and setting is persisted"+ (testing "When the token is valid, it is persisted and its features serve at once"(let [token (tu/random-token)]- (mt/with-dynamic-fn-redefs [token-check/check-token (constantly {:valid true- :status "OK"- :features ["test"]})]- (mt/with-temporary-setting-values [premium-embedding-token nil]+ (mt/with-temporary-setting-values [premium-embedding-token nil]+ (with-store [calls (constantly good-status)](token-check/-set-premium-embedding-token! token)- (is (= token (premium-features/premium-embedding-token))))))))--(deftest set-premium-embedding-token-busts-stale-cache-test- (testing "re-entering a token after renewal revalidates with the store instead of a cached expired verdict (EMB-2341)"- (let [token (tu/random-token)- expired (json/encode {:valid false :status "Token is expired."})- renewed (json/encode {:valid true :status "Token is valid." :features ["sso"]})- state (atom :expired)]- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]- (case @state- :expired {:status 403 :body expired}- :renewed {:status 200 :body renewed}))]- (try- ;; prime the cache with the expired verdict- (is (=? {:valid false :canonical? true} (token-check/check-token token)))+ (is (= token (premium-features/premium-embedding-token)))+ (is (= #{"sandboxes"} (token-check/*token-features*)))+ (is (some? (hash-row token)))+ (is (= 1 @calls)))))))++(deftest set-premium-embedding-token-revalidates-test+ (testing "re-entering a token after renewal revalidates with the store instead of the held expired verdict (EMB-2341)"+ (let [token (tu/random-token)+ state (atom :expired)]+ (mt/with-temporary-setting-values [premium-embedding-token nil]+ (with-store [_ #(case @state+ :expired {:status 403 :body (json/encode {:valid false :status "Token is expired."})}+ :renewed {:valid true :status "Token is valid." :features ["sso"]})]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (is (=? {:valid false :canonical? true} (token-check/refresh-now!))))(reset! state :renewed)- (mt/with-temporary-setting-values [premium-embedding-token nil]- (token-check/-set-premium-embedding-token! token)- (is (= token (premium-features/premium-embedding-token))))- (finally- (token-check/clear-cache!)))))))+ (token-check/-set-premium-embedding-token! token)+ (is (= token (premium-features/premium-embedding-token)))+ (is (= #{"sso"} (token-check/*token-features*))))))))(deftest invalid-token-error-response-test(testing "an invalid token's error response carries its `:error-details`, and nothing else from the ex-data"@@ -1158,3 +918,38 @@:error-details "Token should be 64 hexadecimal characters."}(api-error/response-data data)))(is (= 400 (:status-code data))))))++(deftest refresh-for-a-replaced-token-does-not-overwrite-the-new-one-test+ (testing "a refresh for the old token that finishes after the setter stored a new one leaves the new status held"+ (let [old-token (tu/random-token)+ new-token (tu/random-token)+ fetching (promise)+ release (promise)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token old-token]+ (mt/with-dynamic-fn-redefs [token-check/http-fetch+ (fn [_base-url token _site-uuid]+ (if (= token old-token)+ (do (deliver fetching true)+ @release+ {:status 200 :body (json/encode {:valid true :status "ok" :features ["old"]})})+ {:status 200 :body (json/encode {:valid true :status "ok" :features ["new"]})}))]+ (let [old-refresh (future (token-check/refresh-now!))]+ (is (true? (deref fetching 5000 :timeout)))+ (token-check/-set-premium-embedding-token! new-token)+ (is (= #{"new"} (token-check/*token-features*)))+ (deliver release true)+ (is (=? {:features ["old"]} (deref old-refresh 5000 :timeout)))+ (is (= #{"new"} (token-check/*token-features*)))))))))++(deftest no-app-db-with-a-token-configured-test+ (testing "with a token configured and no app DB, a feature check reads only memory and does not throw"+ (let [token (tu/random-token)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]+ (with-store [_ (constantly good-status)]+ (token-check/refresh-now!)))+ (mt/with-dynamic-fn-redefs [premium-features.settings/premium-embedding-token (constantly token)+ mdb/db-is-set-up? (constantly false)]+ (is (= #{"sandboxes"} (token-check/*token-features*)))+ (reset! @#'token-check/entry nil)+ (is (= #{} (token-check/*token-features*)))+ (is (=? {:valid false :canonical? false} (premium-features/token-status)))))))diff --git a/test/metabase/server/middleware/premium_features_cache_test.clj b/test/metabase/server/middleware/premium_features_cache_test.cljindex fe43ccc9bb8..ddb56ae8347 100644--- a/test/metabase/server/middleware/premium_features_cache_test.clj+++ b/test/metabase/server/middleware/premium_features_cache_test.clj@@ -2,8 +2,11 @@(:require[clojure.test :refer :all][metabase.premium-features.core :as premium-features]+ [metabase.premium-features.test-util :as premium-features.tu]+ [metabase.premium-features.token-check :as token-check][metabase.server.middleware.premium-features-cache :as mw.pf-cache]- [metabase.test :as mt])+ [metabase.test :as mt]+ [metabase.util.json :as json])(:import(java.time Duration Instant)(java.util.concurrent CountDownLatch)))@@ -110,3 +113,30 @@(.countDown start)(run! deref futures)(is (= 1 @calls))))))))++(deftest newer-cookie-refreshes-token-status-test+ (testing "a newer cookie makes an instance that holds no status fetch it from the store"+ (let [entry (var-get #'token-check/entry)+ saved @entry+ calls (atom 0)+ ;; an hour between ticks, so none runs during the test+ started? (premium-features/start-refresher! (* 60 60 1000))]+ (reset! entry nil)+ (try+ (with-redefs [mw.pf-cache/last-known-invalidation (atom nil)]+ (mt/with-temporary-raw-setting-values [premium-embedding-token (premium-features.tu/random-token)]+ (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]+ (swap! calls inc)+ {:status 200+ :body (json/encode {:valid true+ :status "ok"+ :features ["sandboxes"]})})]+ ;; not a feature read: that would tick early on the refresher thread+ (is (nil? @entry))+ (run-middleware (request-with-cookie (str (Instant/now))) {:status 200})+ (is (= 1 @calls))+ (is (true? (premium-features/has-feature? :sandboxes))))))+ (finally+ (when started?+ (premium-features/stop-refresher!))+ (reset! entry saved))))))diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex f18ac746ea0..55bb01deca2 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -755,12 +755,13 @@The token-value binding will contain the random token that was set."[[token-value] & body]- `(let [~token-value (premium-features.test-util/random-token)]- (with-redefs [metabase.premium-features.token-check/check-token- (constantly {:valid true- :status "fake"- :features ["test" "fixture"]- :trial false})]+ `(let [~token-value (premium-features.test-util/random-token)+ status# {:valid true+ :status "fake"+ :features ["test" "fixture"]+ :trial false}]+ (with-redefs [metabase.premium-features.token-check/current-status (constantly status#)+ metabase.premium-features.token-check/fetch-status (constantly status#)](with-temporary-raw-setting-values [:premium-embedding-token ~token-value]~@body))))