Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Refresh the premium token on a per-instance job instead of on read (#85297)

John Swansone04948d09c77
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 9235255485d..905c656cadf 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2047,7 +2047,6 @@
app-db
dependencies
settings
- startup
task
tracing
util
diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.clj
index 4ae91449cbc..e499ae65a03 100644
--- a/src/metabase/core/core.clj
+++ b/src/metabase/core/core.clj
@@ -25,7 +25,7 @@
[metabase.notification.core :as notification]
[metabase.oauth-server.api :as oauth-server.api]
[metabase.plugins.core :as plugins]
- [metabase.premium-features.core :refer [defenterprise]]
+ [metabase.premium-features.core :as premium-features :refer [defenterprise]]
[metabase.sample-data.core :as sample-data]
[metabase.search.embeddings.startup :as embeddings.startup]
[metabase.server.core :as server]
@@ -108,6 +108,7 @@
[]
(log/info "Metabase Shutting Down ...")
(queue/stop-listeners!)
+ (premium-features/stop-refresher!)
(startup/run-shutdown-logic!)
(task/stop-scheduler!)
(server/stop-web-server!)
@@ -205,6 +206,9 @@
;; negatives, but for now there's not much we can do
(mdb/setup-db! :create-sample-content? (not config/is-test?))
(mdb/encrypt-plaintext-columns!)
+ ;; fetch the premium token status now, so that everything below sees this instance's features
+ (premium-features/refresh-at-startup!)
+ (premium-features/start-refresher!)
;; Disable read-only mode if its on during startup.
;; This can happen if a cloud migration process dies during h2 dump.
(when (cloud-migration/read-only-mode)
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index ccb93850eb0..791125584a0 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -38,6 +38,9 @@
TokenStatus
clear-cache!
clear-local-cache!
+ refresh-at-startup!
+ start-refresher!
+ stop-refresher!
token-check-url
transform-metered-as
transform-stats]
diff --git a/src/metabase/premium_features/db.clj b/src/metabase/premium_features/db.clj
index a9375e3c0f2..d67c8c5707c 100644
--- a/src/metabase/premium_features/db.clj
+++ b/src/metabase/premium_features/db.clj
@@ -34,8 +34,3 @@
(t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash
:token_status_hash token-status-hash
:updated_at updated-at}))
-
-(mu/defn delete-token-status-cache!
- "Delete every cached token status."
- []
- (t2/delete! :model/PremiumFeaturesCache))
diff --git a/src/metabase/premium_features/init.clj b/src/metabase/premium_features/init.clj
index ba462c880e2..3708aa217b7 100644
--- a/src/metabase/premium_features/init.clj
+++ b/src/metabase/premium_features/init.clj
@@ -2,5 +2,4 @@
(:require
[metabase.premium-features.core]
[metabase.premium-features.settings]
- [metabase.premium-features.task.clear-token-cache]
[metabase.premium-features.task.send-metering]))
diff --git a/src/metabase/premium_features/rest/api.clj b/src/metabase/premium_features/rest/api.clj
index 53d6a2e03db..9214af4c748 100644
--- a/src/metabase/premium_features/rest/api.clj
+++ b/src/metabase/premium_features/rest/api.clj
@@ -51,15 +51,20 @@
#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
(api.macros/defendpoint :post "/token/refresh"
- "Clear all token caches and re-check the premium features token against the MetaStore.
- Returns the fresh token status. Useful for the frontend after a purchase so that new features
- are recognized without waiting for the cache TTL."
+ "Re-check the premium features token against the MetaStore and return the fresh token status. Useful for the
+ frontend after a purchase so that new features are recognized without waiting for the next scheduled refresh.
+
+ When the check fails, returns an unvalidated status whose `:error-details` says why, so the caller knows the check
+ failed. The status this instance holds is left as it was, so feature checks keep using it."
[]
- (token-check/clear-cache!)
- (invalidate-llm-proxy-token-cache!)
- (-> (api/check-404 (premium-features/token-status))
- response/response
- (assoc-in [:mb/cookies :cookie/premium-features-cache-timestamp] true)))
+ (let [status (try
+ (token-check/refresh-now!)
+ (catch Throwable e
+ (token-check/unvalidated-status (ex-message e))))]
+ (invalidate-llm-proxy-token-cache!)
+ (-> (api/check-404 status)
+ response/response
+ (assoc-in [:mb/cookies :cookie/premium-features-cache-timestamp] true))))
(def ^{:arglists '([request respond raise])} routes
"`/api/premium-features` routes."
diff --git a/src/metabase/premium_features/task/clear_token_cache.clj b/src/metabase/premium_features/task/clear_token_cache.clj
deleted file mode 100644
index a1fd658e9b9..00000000000
--- a/src/metabase/premium_features/task/clear_token_cache.clj
+++ /dev/null
@@ -1,9 +0,0 @@
-(ns metabase.premium-features.task.clear-token-cache
- "Clears the premium features token cache on startup so the first token check always hits MetaStore fresh.
- Without this, the cache table (which survives restarts) could serve stale feature data."
- (:require
- [metabase.premium-features.token-check :as token-check]
- [metabase.startup.core :as startup]))
-
-(defmethod startup/def-startup-logic! ::clear-token-cache [_]
- (token-check/clear-cache!))
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index 454866b28c3..206d947b797 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -9,7 +9,6 @@
[buddy.core.codecs :as codecs]
[buddy.core.hash :as buddy-hash]
[clj-http.client :as http]
- [clojure.core.memoize :as memoize]
[clojure.string :as str]
[diehard.circuit-breaker :as dh.cb]
[diehard.core :as dh]
@@ -31,10 +30,13 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
+ [metabase.util.retry :as retry]
[metabase.util.string :as u.str]
- [potemkin.types :as p]
[toucan2.connection :as t2.conn]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (dev.failsafe CircuitBreakerOpenException FailsafeException TimeoutExceededException)
+ (java.util.concurrent Executors RejectedExecutionException ScheduledExecutorService ThreadFactory TimeUnit)))
(set! *warn-on-reflection* true)
@@ -234,17 +236,13 @@
(log/infof "Checking with the MetaStore to see whether token '%s' is valid..." (u.str/mask token))
(let [{:keys [body status] :as resp} (http-fetch base-url token site-uuid)]
(if (or (http/success? resp) (<= 400 status 499))
- (do (when (http/success? resp)
- (analytics/inc! :metabase-token-check/attempt {:status :success}))
- (or (some-> (parse-status-body body) canonical)
- ;; a 2xx/4xx whose body is missing or undecodable (a proxy's or WAF's error page, a stripped
- ;; body) carries no verdict from the store, so it is transient like a network error: not cached,
- ;; retried under the breaker, and the previous canonical status stands meanwhile
- (throw (ex-info "Token validation provided no response." {:status status}))))
- ;; exceptions are not cached.
- (do (analytics/inc! :metabase-token-check/attempt {:status :failure})
- (throw (ex-info "An unknown error occurred when validating token." {:status status
- :body body}))))))
+ (let [decoded (or (some-> (parse-status-body body) canonical)
+ (throw (ex-info "Token validation provided no response." {:status status})))]
+ (when (http/success? resp)
+ (analytics/inc! :metabase-token-check/attempt {:status :success}))
+ decoded)
+ (throw (ex-info "An unknown error occurred when validating token." {:status status
+ :body body})))))
(defn- metering-url
[token base-url]
@@ -327,67 +325,6 @@
:status "invalid"
:error-details (trs "Token should be a valid 64 hexadecimal character token or an airgap token.")}))))
-#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
-(def ^:dynamic *token-check-happening* "Var to prevent recursive calls to `fetch-token-status`" false)
-
-(p/defprotocol+ TokenChecker
- "Protocol for checking tokens with cache management."
- (-check-token [this token]
- "Check a token and return TokenStatus map. May throw exceptions on failure.")
- (-clear-cache! [this]
- "Clear any caches in this checker and any wrapped checkers, including the shared DB cache.
- Returns nil. Implementations should delegate to wrapped checkers.")
- (-clear-local-cache! [this]
- "Clear only the in-process memoization in this checker and any wrapped checkers. The shared DB cache and the
- DB-hash-validated local cache are left alone, so the next check re-validates against the DB and only refetches
- from the MetaStore if another instance really did get a different token status. Safe to trigger from untrusted
- input. Returns nil."))
-
-(def store-and-airgap-token-checker
- "Creates a basic token checker that handles HTTP requests and airgap tokens.
- No caching, no retries, no grace period - just the core logic."
- (reify TokenChecker
- (-check-token [_ token]
- (decode-token* token))
- (-clear-cache! [_] nil)
- (-clear-local-cache! [_] nil)))
-
-(defn circuit-breaker-token-checker
- "Wraps a token checker with circuit breaker and timeout logic."
- [token-checker {:keys [circuit-breaker timeout-ms lock]
- :or {lock (Object.)}}]
- (let [breaker (dh.cb/circuit-breaker circuit-breaker)]
- (reify TokenChecker
- (-check-token [_ token]
- (when *token-check-happening*
- (throw (ex-info "Token check is being called recursively, there is a good chance some `defenterprise` is causing this"
- {:pass-thru true})))
- ;; important to not count these errors against the circuit breaker. These are not the types of errors we need
- ;; to circuit break. (#65294)
- (when-not (app-db/db-is-set-up?)
- (throw (ex-info "Metabase DB is not yet set up"
- {:cause :token-check/app-db-not-ready})))
- (locking lock
- (binding [*token-check-happening* true]
- (try (dh/with-circuit-breaker breaker
- (dh/with-timeout {:timeout-ms timeout-ms
- :interrupt? true}
- (-check-token token-checker token)))
- (catch dev.failsafe.CircuitBreakerOpenException _e
- (throw (ex-info (tru "Token validation is currently unavailable.")
- {:cause :token-check/circuit-breaker})))
- ;; a timed-out execution has no cause to unwrap — translate it ourselves
- (catch dev.failsafe.TimeoutExceededException _e
- (throw (ex-info (tru "Token validation timed out.") {})))
- ;; other exceptions are wrapped by Diehard in a FailsafeException. Unwrap them before
- ;; rethrowing.
- (catch dev.failsafe.FailsafeException e
- (throw (or (.getCause e) e)))))))
- (-clear-cache! [_]
- (-clear-cache! token-checker))
- (-clear-local-cache! [_]
- (-clear-local-cache! token-checker)))))
-
(defn- hash-token
"SHA-256 hex hash of a token string. Used as cache table key so the raw token never appears in plaintext."
^String [^String token]
@@ -408,11 +345,6 @@
^String [result]
(-> result canonicalize-for-hash json/encode (.getBytes "UTF-8") buddy-hash/sha256 codecs/bytes->hex))
-(defn- read-cache-from-db
- "Read a cached token status hash from the premium_features_token_cache table. Returns nil if not found."
- [token-hash]
- (premium-features.db/token-status-cache token-hash))
-
(defn- write-cache-to-db!
"Upsert a token status hash into the premium_features_token_cache table.
Uses update-first to avoid poisoning PostgreSQL transactions on duplicate key.
@@ -429,11 +361,6 @@
;; Another instance inserted first — update instead.
(premium-features.db/update-token-status-cache! token-hash result-hash now)))))))
-(defn- clear-db-cache!
- "Delete all rows from the premium_features_token_cache table."
- []
- (premium-features.db/delete-token-status-cache!))
-
(defn- extract-locks
"Project a `:meters` map to `{meter-keyword -> boolean}` of `:is-locked` values.
Meters without an `:is-locked` field are dropped."
@@ -451,222 +378,309 @@
(catch Throwable t
(log/warnf "Failed to mirror :locked-meters from token-check response: %s" (ex-message t))))))
-#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
-(def ^:dynamic *testing-only-call-after-refresh*
- "When non-nil, a zero-arg function called after async background refresh completes.
- For testing only — do not use in production."
- nil)
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | REFRESHER |
+;;; +----------------------------------------------------------------------------------------------------------------+
-(defn db-hash-aware-token-checker
- "Wraps a token checker with a DB-hash-aware local cache.
- The DB stores only a hash of the token status (never the actual features). The full token status
- lives exclusively in the local in-memory cache. The DB hash is used to:
- 1. Validate that the local cache is consistent with what other instances see
- 2. Detect when another instance has refreshed and got different features (hash mismatch)
-
- - `soft-ttl`: local entries younger than this are fresh; older triggers async refresh
- - `hard-ttl`: local entries older than this are expired; synchronous fetch required
- - `local-cache`: optional atom for the local cache (default: fresh atom). Exposed for testing.
- - Hash mismatch or cold start always triggers a synchronous fetch."
- [token-checker {:keys [soft-ttl hard-ttl local-cache]}]
- (let [local-cache (or local-cache (atom {}))
- refresh-in-progress? (atom false)]
- (letfn [(do-refresh! [token token-hash]
- (let [result (-check-token token-checker token)]
- ;; a non-authoritative return here is an internal bug: every legitimate inner path
- ;; returns a canonical map or throws
- (when-not (:canonical? result)
- (throw (ex-info "Refusing to cache a non-authoritative token status"
- {:status (:status result)})))
- (let [result-hash (hash-token-status result)
- now (t/instant)]
- (write-cache-to-db! token-hash result-hash)
- (update-locked-meters! result)
- (swap! local-cache assoc token-hash {:result result
- :result-hash result-hash
- :updated-at now})
- result)))]
- (reify TokenChecker
- (-check-token [_ token]
- (if-not (app-db/db-is-set-up?)
- (-check-token token-checker token)
- (let [token-hash (hash-token token)
- local-entry (get @local-cache token-hash)
- db-row (read-cache-from-db token-hash)
- now (t/instant)]
- (if (and local-entry
- db-row
- (= (:result-hash local-entry) (:token_status_hash db-row)))
- ;; Local cache matches DB hash — check TTLs
- (let [age-start (:updated-at local-entry)]
- (cond
- ;; Fresh (< soft-ttl): return local value
- (t/before? now (t/plus age-start soft-ttl))
- (:result local-entry)
-
- ;; Stale (soft..hard): return local + async refresh
- (t/before? now (t/plus age-start hard-ttl))
- (do
- (when (compare-and-set! refresh-in-progress? false true)
- (future
- (try
- (do-refresh! token token-hash)
- (catch Exception e
- (log/errorf "Background premium features refresh failed: %s" (ex-message e)))
- (finally
- (reset! refresh-in-progress? false)
- (when-let [after *testing-only-call-after-refresh*]
- (after))))))
- (:result local-entry))
-
- ;; Expired (> hard-ttl): synchronous refresh
- :else
- (do-refresh! token token-hash)))
- ;; No local cache, no DB row, or hash mismatch: synchronous fetch
- (do-refresh! token token-hash)))))
- (-clear-cache! [_]
- (reset! local-cache {})
- (when (app-db/db-is-set-up?)
- (clear-db-cache!))
- (-clear-cache! token-checker))
- (-clear-local-cache! [_]
- ;; `local-cache` is validated against the DB hash on every check, so it self-corrects; dropping it would
- ;; only force a synchronous MetaStore fetch.
- (-clear-local-cache! token-checker))))))
-
-(defn local-cached-token-checker
- "Wraps a token checker with short-lived in-memory TTL memoization.
- Avoids hitting the DB on every request. Passes through uncached until the app DB is set up, so the
- app-db-not-ready failure is never cached past setup."
- [token-checker {:keys [local-ttl]}]
- (let [cached-check (memoize/ttl
- (fn [token] (-check-token token-checker token))
- :ttl/threshold (.toMillis ^java.time.Duration local-ttl))]
- (reify TokenChecker
- (-check-token [_ token]
- (if (app-db/db-is-set-up?)
- (cached-check token)
- (-check-token token-checker token)))
- (-clear-cache! [_]
- (memoize/memo-clear! cached-check)
- (-clear-cache! token-checker))
- (-clear-local-cache! [_]
- (memoize/memo-clear! cached-check)
- (-clear-local-cache! token-checker)))))
-
-(defn- error-catching-token-checker
- [token-checker]
- (reify TokenChecker
- (-check-token [_ token]
- (try (-check-token token-checker token)
- (catch Exception e
- (when-not (-> e ex-data :cause #{:token-check/circuit-breaker :token-check/app-db-not-ready})
- (log/infof "Error checking token: %s" (ex-message e))
- (analytics/inc! :metabase-token-check/attempt {:status :failure}))
- {:valid false
- :canonical? false
- :status (tru "Unable to validate token")
- :error-details (.getMessage e)})))
- (-clear-cache! [_] (-clear-cache! token-checker))
- (-clear-local-cache! [_] (-clear-local-cache! token-checker))))
+(def ^:private grace-period
+ "How long a status is honored after it was fetched."
+ (t/hours 36))
-#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
-(def ^:dynamic *customize-checker*
- "Dynamic variable allowing for customized token checkers. In the app, we want all of these in place. Only in tests
- should we construct ones without circuit breakers "
- false)
-
-(defn make-checker
- "Make a token checker. Takes a base [[TokenChecker]] token checker and then arguments for the middleware-style
- wrapping [[TokenChecker]] arguments.
-
- `db-hash-local-cache` is optional and only provided during tests."
- [{:keys [base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl db-hash-local-cache]
- :or {base store-and-airgap-token-checker}}]
- (when-not *customize-checker*
- (assert (and base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl)
- "Must provide all arguments for token checker"))
- (cond-> base
- circuit-breaker
- (circuit-breaker-token-checker {:circuit-breaker circuit-breaker
- :timeout-ms timeout-ms})
-
- (and soft-ttl hard-ttl)
- (db-hash-aware-token-checker {:soft-ttl soft-ttl
- :hard-ttl hard-ttl
- :local-cache db-hash-local-cache})
-
- ;; inside the memoize so failure maps are memoized for local-ttl (negative cache; core.memoize
- ;; single-flights values but not thrown exceptions), except before app-DB setup, when the memo
- ;; is skipped; outside db-hash so they never reach the durable cache
- :always
- (error-catching-token-checker)
-
- local-ttl
- (local-cached-token-checker {:local-ttl local-ttl})))
-
-(def token-checker
- "The token checker. Combines http/airgapping validation, circuit breaking, DB-hash-aware caching, and error handling."
- (make-checker {:base store-and-airgap-token-checker
- ;; the window must fit 10 worst-case failures (10s timeout + 5s negative cache
- ;; each ≈ 135s) or slow failures can never open the breaker
- :circuit-breaker {:failure-threshold-ratio-in-period [10 10 (u/seconds->ms 180)]
- :delay-ms (u/seconds->ms 30)
- :success-threshold 1
- :on-open (fn [_] (log/info "Engaging circuit breaker in token check"))
- :on-half-open (fn [_] (log/info "In token check circuit breaker but attempting check"))
- :on-close (fn [_] (log/info "Token Check restored"))}
- :timeout-ms (u/seconds->ms 10)
- :local-ttl (t/seconds 5)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)}))
+;; `{:token-hash, :status, :status-hash, :fetched-at}`, or nil
+(defonce ^:private entry (atom nil))
+
+;; token hash -> promise of `{:status ...}` or `{:error ...}`
+(defonce ^:private in-flight (atom {}))
+
+(defonce ^:private refresher-executor (atom nil))
+
+;; `{:token-hash, :message}` of the latest failed refresh, or nil
+(defonce ^:private last-failure (atom nil))
+
+(def ^:private retry-policy
+ {:max-retries 2
+ :initial-interval-millis 1000
+ :multiplier 2.0
+ :jitter-factor 0.1
+ :max-interval-millis 10000})
+
+(defonce ^:private breaker
+ (dh.cb/circuit-breaker {:failure-threshold 3
+ :delay-ms (u/seconds->ms 30)
+ :on-open (fn [_] (log/info "Engaging circuit breaker in token check"))
+ :on-close (fn [_] (log/info "Token Check restored"))}))
+
+(def ^:private fetch-timeout-ms
+ "Caps a fetch, retries included. A blocked socket read is not interrupted: it ends at the HTTP client's own timeouts."
+ (u/seconds->ms 10))
+
+(defn- fetch-status
+ [token & {:keys [retry?] :or {retry? true}}]
+ (try
+ (dh/with-circuit-breaker breaker
+ (dh/with-timeout {:timeout-ms fetch-timeout-ms
+ :interrupt? true}
+ (if retry?
+ (retry/with-retry (assoc retry-policy
+ :on-retry (fn [_ e] (log/infof "Retrying token check after: %s" (ex-message e))))
+ (decode-token* token))
+ (decode-token* token))))
+ (catch CircuitBreakerOpenException _e
+ (throw (ex-info (tru "Token validation is currently unavailable.") {})))
+ (catch TimeoutExceededException _e
+ (throw (ex-info (tru "Token validation timed out.") {})))
+ ;; Failsafe wraps checked exceptions, such as a `ConnectException`
+ (catch FailsafeException e
+ (throw (or (ex-cause e) e)))))
+
+(defn- install!
+ [token status]
+ ;; the token may have been replaced while its status was being fetched
+ (when (= token (premium-features.settings/premium-embedding-token))
+ (let [token-hash (hash-token token)
+ status-hash (hash-token-status status)]
+ (reset! entry {:token-hash token-hash
+ :status status
+ :status-hash status-hash
+ :fetched-at (t/instant)})
+ (reset! last-failure nil)
+ (try
+ (write-cache-to-db! token-hash status-hash)
+ (catch Throwable e
+ (log/warnf "Failed to record the token status hash: %s" (ex-message e))))
+ (update-locked-meters! status)))
+ status)
+
+(defn- refresh!
+ [fetch-opts]
+ (when-let [token (premium-features.settings/premium-embedding-token)]
+ (let [token-hash (hash-token token)
+ mine (promise)
+ [before] (swap-vals! in-flight (fn [m] (cond-> m (not (contains? m token-hash)) (assoc token-hash mine))))
+ shared (get before token-hash)]
+ (when-not shared
+ (try
+ (deliver mine {:status (install! token (fetch-status token fetch-opts))})
+ (log/info "Refreshed the premium features token status.")
+ (catch Throwable e
+ (log/errorf "Error refreshing the premium features token status: %s" (ex-message e))
+ (analytics/inc! :metabase-token-check/attempt {:status :failure})
+ (reset! last-failure {:token-hash token-hash :message (ex-message e)})
+ (deliver mine {:error e}))
+ (finally
+ (swap! in-flight dissoc token-hash))))
+ (let [{:keys [status error]} @(or shared mine)]
+ (if error
+ (throw error)
+ status)))))
+
+(defn refresh-now!
+ "Fetch the status of the configured token and hold it. Concurrent callers share one fetch. Returns the fresh status,
+ nil when no token is configured, or throws when the fetch fails, in which case the held status is left as it was."
+ []
+ (refresh! {}))
+
+(defn refresh-at-startup!
+ "Make one attempt, without retries, to fetch and hold the status of the configured token. A failure is left for
+ [[tick!]] to retry, not thrown."
+ []
+ (try
+ (refresh! {:retry? false})
+ nil
+ (catch Throwable _e
+ nil)))
(defn clear-cache!
- "Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."
+ "Refresh the token status now; see [[refresh-now!]]. A failure is logged, not thrown, and the held status stands."
+ []
+ (try
+ (refresh-now!)
+ nil
+ (catch Throwable _e
+ nil)))
+
+(defonce ^:private refresh-interval
+ ;; jittered per instance so a fleet started together does not fetch together
+ (t/minus (t/hours 12) (t/seconds (rand-int 3600))))
+
+(defonce ^:private refresh-requested? (atom false))
+
+(defn request-refresh!
+ "Have the next [[tick!]] refresh the token status, for a caller that should not wait for the fetch itself."
+ []
+ (reset! refresh-requested? true)
+ nil)
+
+(defn- shared-status-differs?
+ "Whether the shared hash row for the held status is missing or records a different status, as it does once another
+ instance has refreshed."
+ [{:keys [token-hash status-hash]}]
+ (not= (:token_status_hash (premium-features.db/token-status-cache token-hash)) status-hash))
+
+(defn- refresh-reason
+ [token requested?]
+ (let [{:keys [token-hash fetched-at] :as held} @entry]
+ (cond
+ (not= token-hash (hash-token token))
+ "no status is held"
+
+ requested?
+ "a refresh was requested"
+
+ (t/before? (t/plus fetched-at refresh-interval) (t/instant))
+ "the held status is due for its periodic refresh"
+
+ (shared-status-differs? held)
+ "the shared hash row is missing or records a different status")))
+
+(defn tick!
+ "Refresh the token status if one is due. Never throws, so a scheduler keeps calling it. A requested refresh that
+ fails stays requested."
[]
- (-clear-cache! token-checker))
+ (let [[requested?] (reset-vals! refresh-requested? false)]
+ (try
+ (when-let [token (premium-features.settings/premium-embedding-token)]
+ (when-let [reason (refresh-reason token requested?)]
+ (log/infof "Refreshing the premium features token status: %s." reason)
+ (refresh-now!)))
+ nil
+ (catch Throwable e
+ (when requested?
+ (reset! refresh-requested? true))
+ (log/errorf "Premium features token tick failed: %s" (ex-message e))
+ nil))))
(defn clear-local-cache!
- "Clear only this process's in-memory token memoization; see [[-clear-local-cache!]]. Unlike [[clear-cache!]] this
- never touches the shared DB cache, so it is safe to trigger from an untrusted request."
+ "Refresh the token status now if no status is held for the configured token, or if the shared hash row is missing
+ or records a different status than the one held; otherwise do nothing. Before [[start-refresher!]], only request the refresh,
+ since initialization makes its own. A failure is logged, not thrown."
[]
- (-clear-local-cache! token-checker))
+ (try
+ (when-let [token (premium-features.settings/premium-embedding-token)]
+ (let [held @entry]
+ (when (or (not= (:token-hash held) (hash-token token))
+ (shared-status-differs? held))
+ (if @refresher-executor
+ (clear-cache!)
+ (request-refresh!)))))
+ nil
+ (catch Throwable e
+ (log/errorf "Error checking whether the premium features token status is current: %s" (ex-message e))
+ nil)))
+
+(def ^:private tick-interval-ms (u/minutes->ms 1))
-(defn check-token
- "Public entrypoint to the token checking."
- ([token] (check-token token-checker token))
- ([checker token]
- (-check-token checker token)))
+(defn- safe-tick!
+ []
+ (try
+ (tick!)
+ ;; `scheduleWithFixedDelay` stops rescheduling a task that throws
+ (catch Throwable t
+ (log/errorf "Unexpected error in the premium features token tick; ticking continues: %s" (ex-message t)))))
+
+(defn start-refresher!
+ "Start a daemon thread that calls [[tick!]] every `interval-ms`, a minute by default. Returns `true` if this call
+ started it, `false` if one was already running."
+ ([]
+ (start-refresher! tick-interval-ms))
+ ([interval-ms]
+ (let [exec (Executors/newSingleThreadScheduledExecutor
+ (reify ThreadFactory
+ (newThread [_ r]
+ (doto (Thread. r "premium-features-token-refresher")
+ (.setDaemon true)))))]
+ (if (compare-and-set! refresher-executor nil exec)
+ (do (.scheduleWithFixedDelay exec ^Runnable safe-tick! (long interval-ms) (long interval-ms) TimeUnit/MILLISECONDS)
+ true)
+ (do (.shutdown exec)
+ false)))))
+
+(defn stop-refresher!
+ "Stop the thread started by [[start-refresher!]]."
+ []
+ (when-let [^ScheduledExecutorService exec (first (reset-vals! refresher-executor nil))]
+ (.shutdownNow exec)
+ (when-not (.awaitTermination exec 5 TimeUnit/SECONDS)
+ (log/warn "The premium features token refresher did not stop within 5s."))))
+
+;;; +----------------------------------------------------------------------------------------------------------------+
+;;; | READER |
+;;; +----------------------------------------------------------------------------------------------------------------+
+
+;; hash of the last token [[tick-for-new-token!]] ticked for, so each new token gets one early tick
+(defonce ^:private ticked-for-token-hash (atom nil))
+
+(defn- tick-for-new-token!
+ "Have the refresher thread tick now for a token configured since the last refresh, as when another instance stored
+ it, instead of serving no features until the next scheduled tick. Does not wait for the tick. A token whose refresh
+ has already failed is left to the scheduled ticks."
+ [token-hash]
+ (when-let [^ScheduledExecutorService exec @refresher-executor]
+ (when (and (not= token-hash (:token-hash @last-failure))
+ (not= token-hash (first (reset-vals! ticked-for-token-hash token-hash))))
+ (try
+ (.execute exec ^Runnable safe-tick!)
+ (catch RejectedExecutionException _e
+ nil)))))
+
+(defn- current-status
+ []
+ (when-let [token (premium-features.settings/premium-embedding-token)]
+ (let [{:keys [token-hash status fetched-at]} @entry
+ configured-hash (hash-token token)]
+ (if (not= token-hash configured-hash)
+ (do (tick-for-new-token! configured-hash)
+ nil)
+ (when (t/before? (t/instant) (t/plus fetched-at grace-period))
+ status)))))
+
+(defn unvalidated-status
+ "The status reported for a token whose status could not be validated, with `error-details` saying why."
+ ([]
+ (unvalidated-status (tru "No current token status is held. A refresh may be in progress or failing.")))
+ ([error-details]
+ {:valid false
+ :canonical? false
+ :status (tru "Unable to validate token")
+ :error-details error-details}))
+
+(defn- validated-status
+ "The store's status for `token`, a candidate for the premium token. Throws unless the token is valid."
+ [token]
+ (when (mr/validate [:re AirgapToken] token)
+ (assert-valid-airgap-user-count!))
+ (when-not (or (mr/validate [:re RemoteCheckedToken] token)
+ (mr/validate [:re AirgapToken] token))
+ (throw (ex-info (tru "Token format is invalid.")
+ {:status-code 400
+ :error-details "Token should be 64 hexadecimal characters."
+ :response/keys #{:error-details}})))
+ (let [status (try
+ (fetch-status token)
+ (catch Throwable e
+ (throw (ex-info (tru "Unable to validate token")
+ {:error-details (ex-message e)
+ :status-code 503
+ :response/keys #{:error-details}}
+ e))))]
+ (when-not (:valid status)
+ (throw (ex-info (:status status)
+ {:error-details (:error-details status)
+ :status-code 400
+ :response/keys #{:error-details}})))
+ (log/info "Token is valid.")
+ status))
(defn -set-premium-embedding-token!
"Setter for the [[metabase.premium-features.settings/token-status]] setting."
[new-value]
- ;; validate the new value if we're not unsetting it
(try
- (when (seq new-value)
- (when (mr/validate [:re AirgapToken] new-value)
- (assert-valid-airgap-user-count!))
- (when-not (or (mr/validate [:re RemoteCheckedToken] new-value)
- (mr/validate [:re AirgapToken] new-value))
- (throw (ex-info (tru "Token format is invalid.")
- {:status-code 400
- :error-details "Token should be 64 hexadecimal characters."
- :response/keys #{:error-details}})))
- ;; validate against the store, not a cached verdict — the token may be the same string with a
- ;; renewed subscription behind it
- (clear-cache!)
- (let [decoded (check-token new-value)]
- (when-not (:valid decoded)
- (throw (ex-info (:status decoded)
- {:error-details (:error-details decoded)
- ;; If MetaStore told us the token is invalid, use 400. If some other error occurred, a 503 is
- ;; probably more appropriate.
- :status-code (if (:canonical? decoded)
- 400
- 503)
- :response/keys #{:error-details}}))))
- (log/info "Token is valid."))
- (setting/set-value-of-type! :string :premium-embedding-token new-value)
+ ;; validate the new value if we're not unsetting it
+ (let [status (when (seq new-value)
+ (validated-status new-value))]
+ (setting/set-value-of-type! :string :premium-embedding-token new-value)
+ (when status
+ (install! new-value status)))
((requiring-resolve 'metabase.dependencies.hooks/after-premium-token-update))
(catch Throwable e
(log/errorf "Error setting premium features token: %s" (ex-message e))
@@ -680,58 +694,47 @@
[]
(mr/validate AirgapToken (premium-features.settings/premium-embedding-token)))
-(let [cached-logger (memoize/ttl
- ^{::memoize/args-fn (fn [[token _e]] [token])}
- (fn [_token e]
- (log/errorf "Error validating token: %s" (ex-message e)))
- ;; log every five minutes
- :ttl/threshold (* 1000 60 5))]
- #_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
- (mu/defn ^:dynamic *token-features* :- [:set ms/NonBlankString]
- "Get the features associated with the system's premium features token."
- []
- (try
- ;; an invalid token (e.g. expired) confers no features, even if the token-status response still
- ;; lists them (EMB-2341)
- (or (let [{:keys [valid features]} (some-> (premium-features.settings/premium-embedding-token)
- (check-token))]
- (when valid
- (set features)))
- #{})
- (catch Throwable e
- (when (:pass-thru (ex-data e))
- (throw e))
- (cached-logger (premium-features.settings/premium-embedding-token) e)
- #{}))))
+#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
+(mu/defn ^:dynamic *token-features* :- [:set ms/NonBlankString]
+ "Get the features associated with the system's premium features token."
+ []
+ (try
+ ;; an invalid token (e.g. expired) confers no features, even if the token-status response still
+ ;; lists them (EMB-2341)
+ (let [{:keys [valid features]} (current-status)]
+ (if valid
+ (set features)
+ #{}))
+ (catch Throwable e
+ (log/errorf "Error reading premium features: %s" (ex-message e))
+ #{})))
(defn -token-status
"Getter for the [[metabase.premium-features.settings/token-status]] setting."
[]
- (some-> (premium-features.settings/premium-embedding-token)
- (check-token)))
+ (when-let [token (premium-features.settings/premium-embedding-token)]
+ (or (current-status)
+ (let [{:keys [token-hash message]} @last-failure]
+ (if (and message (= token-hash (hash-token token)))
+ (unvalidated-status message)
+ (unvalidated-status))))))
(mu/defn plan-alias :- [:maybe :string]
"Returns a string representing the instance's current plan, if included in the last token status request."
[]
- (some-> (premium-features.settings/premium-embedding-token)
- (check-token)
- :plan-alias))
+ (:plan-alias (current-status)))
(mu/defn quotas :- [:maybe [:sequential [:map]]]
"Returns a vector of maps for each quota of the subscription."
[]
(clear-cache!)
- (some-> (premium-features.settings/premium-embedding-token)
- (check-token)
- :quotas))
+ (:quotas (current-status)))
(mu/defn meters :- [:maybe :map]
"Returns a map of current metered usage for the subscription."
[]
(clear-cache!)
- (some-> (premium-features.settings/premium-embedding-token)
- (check-token)
- :meters))
+ (:meters (current-status)))
(defn has-any-features?
"True if we have a valid premium features token with ANY features."
@@ -750,11 +753,9 @@
"Returns `true` if the token definitively has `feature`, `false` if it definitively does not, or `nil` if the token
status is indeterminate (e.g., network failure, timeout). Returns `false` (not `nil`) when no token is configured."
[feature]
- (if-let [token (premium-features.settings/premium-embedding-token)]
- (let [result (check-token token)]
- (when (:canonical? result)
- (boolean (and (:valid result)
- (contains? (set (:features result)) (name feature))))))
+ (if (premium-features.settings/premium-embedding-token)
+ (when-let [{:keys [valid features]} (current-status)]
+ (boolean (and valid (contains? (set features) (name feature)))))
false))
(defn ee-feature-error
diff --git a/src/metabase/server/middleware/premium_features_cache.clj b/src/metabase/server/middleware/premium_features_cache.clj
index ea8c747a2d5..8267c18a7a6 100644
--- a/src/metabase/server/middleware/premium_features_cache.clj
+++ b/src/metabase/server/middleware/premium_features_cache.clj
@@ -1,14 +1,13 @@
(ns metabase.server.middleware.premium-features-cache
"Ring middleware to propagate premium-features cache invalidation across instances via a cookie.
When one instance refreshes features (via POST /api/premium-features/token/refresh), it sets a cookie
- with the current timestamp. Other instances see this cookie and drop their in-process token caches so
- the next feature check re-validates against the shared DB cache, and only re-fetches from the MetaStore
- if that shows the other instance really did get a different token status.
+ with the current timestamp. Other instances see this cookie and compare the status they hold with the shared hash
+ row in the app DB, refreshing their own token status from the MetaStore only when no status is held or the row
+ is missing or differs. A cookie therefore costs at most an app DB read unless another instance really has a newer status.
The cookie is client-controlled and arrives on unauthenticated requests, so it is a hint, never an
authority: it must parse as a timestamp, must not be in the future, must be newer than the last
- invalidation this instance saw, and can only ever clear this instance's in-memory caches. It never deletes
- the deployment's shared DB cache."
+ invalidation this instance saw, and can only ever trigger a refresh of this instance's own token status."
(:require
[metabase.premium-features.core :as premium-features]
[metabase.util.log :as log]
@@ -31,7 +30,7 @@
(def ^:private last-known-invalidation
"The newest invalidation this instance has seen or emitted, as an `Instant`. Any cookie at or before it is ignored,
- so a given cookie value can trigger a local clear at most once."
+ so a given cookie value can trigger a local refresh at most once."
(atom nil))
(defn- parse-cookie-timestamp
@@ -59,12 +58,12 @@
(not= old new)))
(defn- check-and-invalidate-features-cache
- "If the request cookie carries a valid timestamp newer than the last invalidation this instance knows about, drop
- this instance's in-process token caches. Never touches the shared DB cache."
+ "If the request cookie carries a valid timestamp newer than the last invalidation this instance knows about, have
+ this instance refresh its token status if it is out of date."
[request]
(when-let [cookie-timestamp (parse-cookie-timestamp (get-in request [:cookies cookie-name :value]))]
(when (advance-last-known-invalidation! cookie-timestamp)
- (log/info "Premium features cookie indicates local cache is out of date. Clearing...")
+ (log/info "Premium features cookie indicates the local token status may be out of date. Checking...")
(premium-features/clear-local-cache!))))
(defn- maybe-set-premium-features-cookie
@@ -82,10 +81,10 @@
response))
(defn wrap-premium-features-cache-check
- "Middleware that checks if the local premium features cache needs to be invalidated based on a cookie.
+ "Middleware that checks if the local premium features token status needs a refresh based on a cookie.
When premium features are refreshed on one instance, the API sets a cookie with the current timestamp.
- This middleware checks that cookie on subsequent requests and clears the local cache if needed,
+ This middleware checks that cookie on subsequent requests and refreshes the local status if it is out of date,
ensuring consistency in multi-instance deployments."
[handler]
(fn [request respond raise]
diff --git a/test/metabase/premium_features/rest/api_test.clj b/test/metabase/premium_features/rest/api_test.clj
index 489310afc96..37b6b848224 100644
--- a/test/metabase/premium_features/rest/api_test.clj
+++ b/test/metabase/premium_features/rest/api_test.clj
@@ -4,8 +4,11 @@
[clj-http.cookies :as cookies]
[clojure.test :refer :all]
[metabase.premium-features.core :as premium-features]
+ [metabase.premium-features.test-util :as premium-features.tu]
[metabase.premium-features.token-check :as token-check]
- [metabase.test :as mt])
+ [metabase.test :as mt]
+ [metabase.util.json :as json]
+ [metabase.util.retry :as retry])
(:import
(clojure.lang ExceptionInfo)))
@@ -33,23 +36,72 @@
(deftest post-token-refresh-test
(testing "POST /api/premium-features/token/refresh"
- (testing "clears cache and returns fresh token status"
- (let [cleared? (atom false)]
- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)
- premium-features/premium-embedding-token (constantly nil)
- token-check/clear-cache! (fn [] (reset! cleared? true))]
+ (testing "refreshes and returns the fresh token status"
+ (let [refreshed? (atom false)]
+ (mt/with-dynamic-fn-redefs [premium-features/premium-embedding-token (constantly nil)
+ token-check/refresh-now! (fn []
+ (reset! refreshed? true)
+ fake-token-status)]
(is (=? (dissoc fake-token-status :trial)
(mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))
- (is (true? @cleared?))))))
+ (is (true? @refreshed?))))))
(testing "requires superusers"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 "premium-features/token/refresh"))))
(testing "returns 404 if no token is set"
- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly nil)
+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly nil)
premium-features/premium-embedding-token (constantly nil)]
(is (= "Not found."
(mt/user-http-request :crowberto :post 404 "premium-features/token/refresh"))))))
+(deftest post-token-refresh-returns-fresh-status-test
+ (testing "POST /api/premium-features/token/refresh fetches from the store and returns what it said"
+ (let [entry (var-get #'token-check/entry)
+ saved @entry
+ status (atom {:valid true :status "ok" :features ["sandboxes"]})]
+ (try
+ (mt/with-temp-env-var-value! [mb-premium-embedding-token nil]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (premium-features.tu/random-token)]
+ (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _] {:status 200 :body (json/encode @status)})]
+ (is (=? {:valid true :features ["sandboxes"]}
+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))
+ (reset! status {:valid true :status "ok" :features ["sandboxes" "audit-app"]})
+ (is (=? {:valid true :features ["sandboxes" "audit-app"]}
+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))
+ (is (= #{"sandboxes" "audit-app"} (token-check/*token-features*))))))
+ (finally
+ (reset! entry saved))))))
+
+(deftest post-token-refresh-reports-a-failed-check-test
+ (testing "POST /api/premium-features/token/refresh reports a failed store call, while feature checks keep the held status"
+ (let [entry (var-get #'token-check/entry)
+ saved @entry
+ fail? (atom false)]
+ (try
+ (mt/with-temp-env-var-value! [mb-premium-embedding-token nil]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (premium-features.tu/random-token)]
+ (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
+ (if @fail?
+ (throw (ex-info "network failure!" {}))
+ {:status 200
+ :body (json/encode {:valid true
+ :status "ok"
+ :features ["sandboxes"]})}))]
+ (is (=? {:valid true :features ["sandboxes"]}
+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))
+ (reset! fail? true)
+ (binding [retry/*test-time-config-hook* #(assoc % :initial-interval-millis 1 :max-interval-millis 2)]
+ (is (= {:valid false
+ :canonical? false
+ :status "Unable to validate token"
+ :error-details "network failure!"}
+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh"))))
+ (is (premium-features/has-feature? :sandboxes))
+ (is (=? {:valid true :features ["sandboxes"]} (premium-features/token-status))))))
+ (finally
+ (.close ^dev.failsafe.CircuitBreaker @#'token-check/breaker)
+ (reset! entry saved))))))
+
(deftest post-token-refresh-invalidates-llm-proxy-cache-test
(testing "POST /api/premium-features/token/refresh invalidates the AI service token cache when explicitly configured"
(let [token "SOME_RANDOM_TOKEN"]
@@ -59,7 +111,7 @@
(mt/with-temporary-setting-values [llm-proxy-base-url "https://proxy.example.com/llm/"
ai-service-base-url "https://ai-service.example.com/"]
(let [request* (atom nil)]
- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)
+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly fake-token-status)
http/post (fn [url request-options]
(reset! request* [url request-options])
{:status 200})]
@@ -68,7 +120,7 @@
(is (= (str "https://ai-service.example.com/v1/invalidate-token-cache/" token) url))
(is (false? (:throw-exceptions opts))))))))))))
(testing "POST /api/premium-features/token/refresh does not invalidate the AI service cache when it is not configured"
- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)
+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly fake-token-status)
premium-features/premium-embedding-token (constantly "proxy-token")
http/post (fn [& _]
(throw (ex-info "should not be called" {})))]
@@ -79,7 +131,7 @@
(testing "POST /api/premium-features/token/refresh sets the premium-features-last-updated cookie"
;; user-real-request hits a real Jetty server; handler thread doesn't inherit *local-redefs*.
#_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
- (with-redefs [premium-features/token-status (constantly fake-token-status)
+ (with-redefs [token-check/refresh-now! (constantly fake-token-status)
premium-features/premium-embedding-token (constantly nil)]
(let [cs (cookies/cookie-store)]
(mt/user-real-request :crowberto :post 200 "premium-features/token/refresh"
@@ -100,7 +152,7 @@
(mt/with-premium-features #{:metabase-ai-managed}
(mt/with-temp-env-var-value! [mb-premium-embedding-token nil]
(mt/with-temporary-raw-setting-values [premium-embedding-token token]
- (mt/with-dynamic-fn-redefs [premium-features/token-status (constantly fake-token-status)
+ (mt/with-dynamic-fn-redefs [token-check/refresh-now! (constantly fake-token-status)
http/post (fn [url opts] (reset! request* [url opts]) {:status 200})]
(testing "a stored base URL gets no more than llm-allowed-networks allows"
(mt/with-temporary-setting-values [ai-service-base-url "http://10.0.0.1/"]
diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.clj
index 8c53a64330e..3e62b95b41c 100644
--- a/test/metabase/premium_features/token_check_test.clj
+++ b/test/metabase/premium_features/token_check_test.clj
@@ -4,21 +4,20 @@
[clj-http.core :as http.core]
[clojure.test :refer :all]
[java-time.api :as t]
- [mb.hawk.parallel]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.core :as mdb]
[metabase.premium-features.core :as premium-features]
[metabase.premium-features.db :as premium-features.db]
- [metabase.premium-features.task.clear-token-cache]
+ [metabase.premium-features.settings :as premium-features.settings]
[metabase.premium-features.test-util :as tu]
[metabase.premium-features.token-check :as token-check]
- [metabase.startup.core :as startup]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
[metabase.util :as u]
[metabase.util.api-error :as api-error]
[metabase.util.json :as json]
[metabase.util.malli.registry :as mr]
+ [metabase.util.retry :as retry]
[methodical.core :as methodical]
[toucan2.core :as t2]
[toucan2.pipeline :as t2.pipeline]))
@@ -27,480 +26,610 @@
(use-fixtures :once (fixtures/initialize :db))
-(defn- fresh-checker
- "A token checker with production TTLs and no circuit breaker.
- The global checker's breaker is shared across the JVM, and one tripped by an earlier test would fail every check here."
- []
- (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/seconds 5)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)})))
-
-(deftest log-tests
- (let [token (tu/random-token)
- print-token (apply str (concat (take 4 token) "..." (take-last 4 token)))]
- (testing "Do not log the token (#18249)"
- (mt/with-log-messages-for-level [messages :info]
- (token-check/check-token (fresh-checker) token)
- (let [logs (mapv :message (messages))]
- (is (every? (complement #(re-find (re-pattern token) %)) logs))
- (is (= 1 (count (filter #(re-find (re-pattern print-token) %) logs)))))))))
-
-(deftest fetch-token-caches-invalid-responses
- (testing "For 4XX responses, the result is cached"
- (let [call-count (atom 0)
- token (tu/random-token)]
- (binding [http/request (fn [& _]
- (swap! call-count inc)
- {:status 400 :body "{\"valid\": false, \"status\": \"fake\"}"})]
- (let [checker (fresh-checker)]
- (dotimes [_ 10] (token-check/check-token checker token)))
- (is (= 1 @call-count))))))
-
-(deftest fetch-token-does-not-cache-exceptions
- (let [call-count (atom 0)
- token (tu/random-token)
- response (atom :error)
- ;; no circuit breaker (carries state between blocks) and no local-ttl (it negative-caches
- ;; failures) — this pins that the DURABLE tiers never store them
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:soft-ttl (t/minutes 1)
- :hard-ttl (t/minutes 2)}))]
+(defn- do-with-clean-refresher
+ [thunk]
+ (let [entry @#'token-check/entry
+ breaker @#'token-check/breaker
+ saved @entry]
+ (reset! entry nil)
+ (reset! @#'token-check/refresh-requested? false)
+ (reset! @#'token-check/last-failure nil)
+ (reset! @#'token-check/ticked-for-token-hash nil)
+ (.close ^dev.failsafe.CircuitBreaker breaker)
+ (try
+ (binding [retry/*test-time-config-hook* #(assoc % :initial-interval-millis 1 :max-interval-millis 2)]
+ (thunk))
+ (finally
+ (.close ^dev.failsafe.CircuitBreaker breaker)
+ (reset! entry saved)))))
+
+(use-fixtures :each do-with-clean-refresher)
+
+(defn- do-with-store
+ [respond f]
+ (let [calls (atom 0)]
(mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- (case @response
- :error (throw (ex-info "kaboom" {:ka :boom}))
- :500 {:status 500}
- :400-empty {:status 400}
- :400-html {:status 400 :body "<html>Bad Gateway</html>"}
- :200-empty {:status 200}))]
- (testing "For timeouts, 5XX errors, etc. we don't cache the result"
- (dotimes [_ 5] (token-check/check-token checker token))
- (is (= 5 @call-count)))
- (testing "Does not cache 500 responses"
- (reset! call-count 0)
- (reset! response :500)
- (dotimes [_ 5] (token-check/check-token checker token))
- (is (= 5 @call-count)))
- (testing "A 4XX with no body is not a real answer from the store — treated as transient, not cached"
- (reset! call-count 0)
- (reset! response :400-empty)
- (dotimes [_ 5] (token-check/check-token checker token))
- (is (= 5 @call-count))
- (is (= {:valid false
- :canonical? false
- :status "Unable to validate token"
- :error-details "Token validation provided no response."}
- (token-check/check-token checker token))))
- (testing "An unparseable body (e.g. a proxy's error page) is transient, not cached, and surfaces a legible error"
- (reset! call-count 0)
- (reset! response :400-html)
- (dotimes [_ 5] (token-check/check-token checker token))
- (is (= 5 @call-count))
- (is (=? {:canonical? false
- :error-details "Token validation provided no response."}
- (token-check/check-token checker token))))
- (testing "A 2XX with no body is transient, not cached, and surfaces a legible error"
- (reset! call-count 0)
- (reset! response :200-empty)
- (dotimes [_ 5] (token-check/check-token checker token))
- (is (= 5 @call-count))
- (is (=? {:canonical? false
- :error-details "Token validation provided no response."}
- (token-check/check-token checker token)))))))
-
-(deftest not-found-test
- (mt/with-log-level :fatal
- (is (=? {:valid false, :status "Token does not exist."}
- (token-check/check-token (fresh-checker) (tu/random-token))))))
-
-(deftest expired-token-status-is-authoritative-test
- (testing "a 403 with a body is a real answer from the store: stored faithfully, marked canonical (EMB-2341)"
- (let [token (tu/random-token)
- status {:valid false
+ (swap! calls inc)
+ (let [response (respond)]
+ (if (contains? response :valid)
+ {:status 200 :body (json/encode response)}
+ response)))]
+ (f calls))))
+
+(defmacro ^:private with-store
+ {:style/indent 1, :clj-kondo/lint-as 'clojure.core/let}
+ [[calls-binding respond] & body]
+ `(do-with-store ~respond (fn [~calls-binding] ~@body)))
+
+(defn- hash-row [token]
+ (t2/select-one :model/PremiumFeaturesCache :token_hash (#'token-check/hash-token token)))
+
+(deftest successful-refresh-populates-entry-and-hash-row-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [calls (constantly {:valid true :status "ok" :features ["sandboxes" "audit-app"]})]
+ (testing "before any refresh there are no features"
+ (is (= #{} (token-check/*token-features*))))
+ (is (=? {:valid true :features ["sandboxes" "audit-app"]}
+ (token-check/refresh-now!)))
+ (is (= 1 @calls))
+ (is (= #{"sandboxes" "audit-app"} (token-check/*token-features*)))
+ (is (premium-features/has-feature? :sandboxes))
+ (is (=? {:valid true :status "ok"} (premium-features/token-status)))
+ (is (some? (hash-row token)))))))
+
+(defn- age-entry!
+ [age]
+ (swap! @#'token-check/entry update :fetched-at t/minus age))
+
+(def ^:private good-status {:valid true :status "ok" :features ["sandboxes"]})
+
+(deftest failed-refresh-leaves-held-status-alone-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!))
+ (let [row (hash-row token)]
+ (with-store [calls #(throw (ex-info "network failure!" {}))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"network failure!" (token-check/refresh-now!)))
+ (is (= 3 @calls) "the refresh retries before giving up")
+ (is (= #{"sandboxes"} (token-check/*token-features*)) "the previously held features keep serving")
+ (is (= row (hash-row token)) "the hash row is untouched"))))))
+
+(deftest failed-first-refresh-holds-nothing-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [_ #(throw (ex-info "network failure!" {}))]
+ (is (thrown? clojure.lang.ExceptionInfo (token-check/refresh-now!)))
+ (is (= #{} (token-check/*token-features*)))
+ (is (nil? (hash-row token)))
+ (is (=? {:valid false :canonical? false :status "Unable to validate token"}
+ (premium-features/token-status)))))))
+
+(deftest transient-responses-are-never-held-test
+ (doseq [[description response] {"a 5xx" {:status 500}
+ "a 2xx with no body" {:status 200}
+ "a bare 403" {:status 403}
+ "a bare 404" {:status 404}
+ "a proxy's HTML error page" {:status 400 :body "<html>Bad Gateway</html>"}
+ "an empty object" {:status 200 :body "{}"}
+ "a non-boolean valid" {:status 200 :body "{\"valid\":\"yes\",\"status\":\"ok\"}"}
+ "a blank status" {:status 200 :body "{\"valid\":false,\"status\":\"\"}"}
+ "features that aren't a list" {:status 200 :body "{\"valid\":true,\"status\":\"ok\",\"features\":\"sso\"}"}
+ "a JSON value, not an object" {:status 200 :body "[1,2,3]"}}]
+ (testing (format "%s carries no verdict from the store: the refresh fails and the held status stands" description)
+ (reset! @#'token-check/entry nil)
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!))
+ (with-store [_ (constantly response)]
+ (is (thrown? clojure.lang.ExceptionInfo (token-check/refresh-now!)))
+ (is (= #{"sandboxes"} (token-check/*token-features*))))))))
+
+(deftest rejection-with-a-body-is-authoritative-test
+ (testing "a 4xx carrying a status is the store's answer: it replaces the held status, and confers no features (EMB-2341)"
+ (let [expired {:valid false
:status "Token is expired."
- :features ["sso" "snippet-collections" "database-routing"]
+ :features ["sso" "snippet-collections"]
:plan-alias "pro-self-hosted"
- :trial false
- :valid-thru "2026-08-25T12:00:00Z"}
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/seconds 5)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _] {:status 403 :body (json/encode status)})]
- (is (= (assoc status :canonical? true)
- (token-check/check-token checker token))))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest invalid-token-in-2xx-is-authoritative-test
- (testing "an invalid verdict in a 2xx is a real answer: cached, and features gated the same as a 4xx"
- (let [token (tu/random-token)
- call-count (atom 0)
- body (json/encode {:valid false :status "Token is expired." :features ["sso"]})
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/seconds 5)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- {:status 200 :body body})]
- (is (= {:valid false :status "Token is expired." :features ["sso"] :canonical? true}
- (token-check/check-token checker token)))
- (token-check/check-token checker token)
- (is (= 1 @call-count) "authoritative answers are cached"))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest hung-request-times-out-test
- (testing "a request that never returns times out with a legible error and is not cached"
- (let [token (tu/random-token)
- call-count (atom 0)
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker
- {:base (reify token-check/TokenChecker
- (-check-token [_ _]
- (swap! call-count inc)
- (Thread/sleep 5000))
- (-clear-cache! [_])
- (-clear-local-cache! [_]))
- :circuit-breaker {:failure-threshold-ratio-in-period [10 10 1000]
- :delay-ms 50
- :success-threshold 1}
- :timeout-ms 100
- :local-ttl (t/millis 1)
- :soft-ttl (t/minutes 1)
- :hard-ttl (t/minutes 2)}))]
- (try
- (is (= {:valid false
- :canonical? false
- :status "Unable to validate token"
- :error-details "Token validation timed out."}
- (token-check/check-token checker token)))
- (Thread/sleep 5) ;; expire the local memoize layer
- (token-check/check-token checker token)
- (is (= 2 @call-count) "timeouts are not cached")
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest transient-failures-are-negative-cached-test
- ;; timing rule: asserting behavior WITHIN a TTL window needs a TTL generous enough that test
- ;; overhead can't overrun it (10s); sleeping PAST a tight TTL is deterministic (sleep ≥ duration)
- (testing "a transient failure is memoized for local-ttl — an outage costs one attempt per TTL, not one per feature check"
- (let [token (tu/random-token)
- call-count (atom 0)
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/seconds 10)
- :soft-ttl (t/minutes 1)
- :hard-ttl (t/minutes 2)}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- (throw (ex-info "network failure!" {})))]
- (dotimes [_ 5] (token-check/check-token checker token))
- (is (= 1 @call-count) "failures within the TTL are served from the negative cache")
- (is (=? {:valid false :canonical? false :error-details "network failure!"}
- (token-check/check-token checker token))))
- (finally
- (token-check/-clear-cache! checker)))))
- (testing "the negative cache expires with the TTL — failures are retried"
- (let [token (tu/random-token)
- call-count (atom 0)
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/millis 50)
- :soft-ttl (t/minutes 1)
- :hard-ttl (t/minutes 2)}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- (throw (ex-info "network failure!" {})))]
- (token-check/check-token checker token)
- (is (= 1 @call-count))
+ :valid-thru "2026-08-25T12:00:00Z"}]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!))
+ (with-store [_ (constantly {:status 403 :body (json/encode expired)})]
+ (is (= (assoc expired :canonical? true) (token-check/refresh-now!)))
+ (is (= #{} (token-check/*token-features*)))
+ (is (false? (premium-features/has-feature? :sso)))
+ (is (false? (premium-features/canonically-has-feature? :sso)))
+ (is (= "pro-self-hosted" (premium-features/plan-alias)))
+ (is (= (assoc expired :canonical? true) (premium-features/token-status))))))))
+
+(deftest invalid-status-in-2xx-confers-no-features-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ (constantly {:valid false :status "Token is expired." :features ["sso"]})]
+ (token-check/refresh-now!)
+ (is (= #{} (token-check/*token-features*)))
+ (is (false? (premium-features/canonically-has-feature? :sso))))))
+
+(deftest feature-check-reads-no-app-db-test
+ (doseq [held? [false true]]
+ (testing (if held? "with a held status" "with no held status")
+ (reset! @#'token-check/entry nil)
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (when held?
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!)))
+ ;; the first read may run the settings cache's own once-a-minute staleness check
+ (premium-features/has-feature? :sandboxes)
+ (with-store [calls (constantly good-status)]
+ (t2/with-call-count [call-count]
+ (dotimes [_ 10]
+ (is (= held? (premium-features/has-feature? :sandboxes)))
+ (premium-features/token-status))
+ (is (zero? (call-count))))
+ (is (zero? @calls)))))))
+
+(deftest feature-check-does-not-wait-for-a-refresh-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!))
+ (let [fetching (promise)
+ release (promise)]
+ (with-store [_ (fn []
+ (deliver fetching true)
+ @release
+ {:valid true :status "ok" :features ["audit-app"]})]
+ (let [refresh (future (token-check/refresh-now!))]
+ (is (true? (deref fetching 5000 :timeout)))
+ (is (= #{"sandboxes"} (token-check/*token-features*)) "the held status serves while the refresh is in flight")
+ (deliver release true)
+ (is (=? {:features ["audit-app"]} (deref refresh 5000 :timeout)))
+ (is (= #{"audit-app"} (token-check/*token-features*))))))))
+
+(deftest concurrent-refreshes-share-one-fetch-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (let [fetching (promise)
+ release (promise)]
+ (with-store [calls (fn []
+ (deliver fetching true)
+ @release
+ good-status)]
+ (let [first-refresh (future (token-check/refresh-now!))
+ _ (deref fetching 5000 :timeout)
+ others (mapv (fn [_] (future (token-check/refresh-now!))) (range 7))]
(Thread/sleep 100)
- (token-check/check-token checker token)
- (is (= 2 @call-count)))
- (finally
- (token-check/-clear-cache! checker)))))
- (testing "concurrent misses share a single attempt (core.memoize single-flights values, and failures are values here)"
- (let [token (tu/random-token)
- call-count (atom 0)
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/seconds 10)
- :soft-ttl (t/minutes 1)
- :hard-ttl (t/minutes 2)}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- (Thread/sleep 100)
- (throw (ex-info "network failure!" {})))]
- (let [checks (mapv (fn [_] (future (token-check/check-token checker token))) (range 8))]
- (is (every? #(false? (:valid %)) (mapv deref checks)))
- (is (= 1 @call-count) "8 concurrent callers, one network attempt")))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest bare-rejection-is-transient-test
- (doseq [http-status [403 404]]
- (testing (format "a bare %d (no body) carries no verdict from the store: a proxy may have stripped it, so it is transient, not cached"
- http-status)
- (let [token (tu/random-token)
- call-count (atom 0)
- ;; no local-ttl: it negative-caches failures, and this pins that the durable tiers never store them
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:soft-ttl (t/minutes 1)
- :hard-ttl (t/minutes 2)}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- {:status http-status})]
- (dotimes [_ 5] (token-check/check-token checker token))
- (is (= 5 @call-count) "not cached: every check goes back to the store")
- (is (=? {:valid false
- :canonical? false
- :error-details "Token validation provided no response."}
- (token-check/check-token checker token))))
- (finally
- (token-check/-clear-cache! checker)))))))
-
-(deftest malformed-status-body-is-transient-test
- (doseq [[description body] {"an empty object" "{}"
- "an unrelated object" "{\"foo\":1}"
- "a non-boolean valid" "{\"valid\":\"yes\",\"status\":\"ok\"}"
- "a blank status" "{\"valid\":false,\"status\":\"\"}"
- "features that aren't a list" "{\"valid\":true,\"status\":\"ok\",\"features\":\"sso\"}"
- "a JSON value, not an object" "[1,2,3]"}]
- (testing (format "%s decodes as JSON but is not a token status: transient, never stamped canonical or cached" description)
- (let [token (tu/random-token)
- call-count (atom 0)
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:soft-ttl (t/minutes 1)
- :hard-ttl (t/minutes 2)}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- {:status 200 :body body})]
- (dotimes [_ 3] (token-check/check-token checker token))
- (is (= 3 @call-count) "not cached")
- (is (=? {:valid false
- :canonical? false
- :error-details "Token validation provided no response."}
- (token-check/check-token checker token))))
- (finally
- (token-check/-clear-cache! checker)))))))
-
-(deftest do-refresh-refuses-non-canonical-test
- (testing "the caching layer refuses to hold on to a result not marked :canonical?"
- (let [token (tu/random-token)
- call-count (atom 0)
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker
- {:base (reify token-check/TokenChecker
- (-check-token [_ _]
- (swap! call-count inc)
- {:valid true :status "fake" :features ["sso"]})
- (-clear-cache! [_])
- (-clear-local-cache! [_]))
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)}))]
+ (deliver release true)
+ (is (every? #(= (assoc good-status :canonical? true) (deref % 5000 :timeout))
+ (cons first-refresh others)))
+ (is (= 1 @calls)))))))
+
+(deftest grace-period-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ (testing "a status younger than 36h is honored"
+ (age-entry! (t/hours 35))
+ (is (= #{"sandboxes"} (token-check/*token-features*)))
+ (is (true? (premium-features/canonically-has-feature? :sandboxes))))
+ (testing "an older one is not"
+ (age-entry! (t/hours 2))
+ (is (= #{} (token-check/*token-features*)))
+ (is (nil? (premium-features/canonically-has-feature? :sandboxes)))
+ (is (=? {:valid false :canonical? false :status "Unable to validate token"}
+ (premium-features/token-status))))
+ (is (= 1 @calls) "reading never fetches"))))
+
+(deftest no-entry-before-app-db-setup-test
+ (testing "with no held status and no app DB, a feature check yields nothing and does not throw"
+ (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly false)]
+ (is (= #{} (token-check/*token-features*)))
+ (is (false? (premium-features/has-feature? :sandboxes))))))
+
+(deftest status-for-another-token-is-never-served-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!)))
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (is (= #{} (token-check/*token-features*)))
+ (is (=? {:valid false :canonical? false} (premium-features/token-status)))))
+
+(deftest airgap-refresh-makes-no-http-call-test
+ (let [airgap-status {:valid true :status "Token is valid" :features ["sandboxes"] :max-users 10}]
+ (mt/with-dynamic-fn-redefs [token-check/decode-airgap-token (constantly airgap-status)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token "airgap_fake.token"]
+ (with-store [calls #(throw (ex-info "no HTTP call expected" {}))]
+ (is (= (assoc airgap-status :canonical? true) (token-check/refresh-now!)))
+ (is (= #{"sandboxes"} (token-check/*token-features*)))
+ (is (zero? @calls)))))))
+
+(deftest refresh-mirrors-locked-meters-test
+ (mt/with-temporary-setting-values [locked-meters {}]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (let [response (atom nil)]
+ (with-store [_ #(deref response)]
+ (testing "a status with :meters writes through"
+ (reset! response {:valid true :status "ok" :meters {:transform-basic-runs {:is-locked true}
+ :transform-advanced-runs {:is-locked false}}})
+ (token-check/refresh-now!)
+ (is (= {:transform-basic-runs true :transform-advanced-runs false} (premium-features/locked-meters))))
+ (testing "a status without :meters leaves the setting alone"
+ (reset! response {:valid true :status "ok"})
+ (token-check/refresh-now!)
+ (is (= {:transform-basic-runs true :transform-advanced-runs false} (premium-features/locked-meters))))
+ (testing "an empty :meters unlocks everything"
+ (reset! response {:valid true :status "ok" :meters {}})
+ (token-check/refresh-now!)
+ (is (= {} (premium-features/locked-meters))))
+ (testing "a failed refresh leaves the setting alone (a network blip must never unlock)"
+ (reset! response {:valid true :status "ok" :meters {:transform-basic-runs {:is-locked true}}})
+ (token-check/refresh-now!)
+ (reset! response {:status 503})
+ (is (thrown? clojure.lang.ExceptionInfo (token-check/refresh-now!)))
+ (is (= {:transform-basic-runs true} (premium-features/locked-meters)))))))))
+
+(deftest breaker-stops-fetching-after-consecutive-failures-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls #(throw (ex-info "network failure!" {}))]
+ (dotimes [_ 3]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"network failure!" (token-check/refresh-now!))))
+ (reset! calls 0)
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Token validation is currently unavailable"
+ (token-check/refresh-now!)))
+ (is (zero? @calls)))))
+
+(deftest fetch-is-capped-by-a-timeout-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-redefs [token-check/fetch-timeout-ms 100]
+ (with-store [_ #(do (Thread/sleep 5000) good-status)]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Token validation timed out"
+ (token-check/refresh-now!)))))))
+
+(deftest checked-fetch-exceptions-are-unwrapped-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ #(throw (java.net.ConnectException. "Connection refused"))]
+ (is (thrown-with-msg? java.net.ConnectException #"Connection refused" (token-check/refresh-now!))))))
+
+(deftest token-status-reports-the-latest-failure-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [_ #(throw (java.net.ConnectException. "Connection refused"))]
+ (premium-features/clear-cache!)
+ (is (=? {:valid false :canonical? false :error-details "Connection refused"}
+ (premium-features/token-status))))
+ (testing "a successful refresh clears it"
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!))
+ (is (=? {:valid true :status "ok"} (premium-features/token-status)))))
+ (testing "another token's failure is not reported"
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [_ #(throw (java.net.ConnectException. "Connection refused"))]
+ (premium-features/clear-cache!)))
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (is (=? {:valid false :canonical? false :error-details #"No current token status is held.*"}
+ (premium-features/token-status)))))))
+
+(deftest clear-cache-refreshes-and-never-throws-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ (constantly good-status)]
+ (is (nil? (premium-features/clear-cache!)))
+ (is (= #{"sandboxes"} (token-check/*token-features*))))
+ (with-store [_ #(throw (ex-info "network failure!" {}))]
+ (is (nil? (premium-features/clear-cache!)))
+ (is (= #{"sandboxes"} (token-check/*token-features*))))))
+
+(defn- do-with-refresher-started!
+ [thunk]
+ ;; an hour between ticks, so none runs during the test
+ (premium-features/start-refresher! (u/hours->ms 1))
+ (try
+ (thunk)
+ (finally
+ (premium-features/stop-refresher!))))
+
+(deftest tick-with-no-entry-held-fetches-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (token-check/tick!)
+ (is (= 1 @calls))
+ (is (premium-features/has-feature? :sandboxes)))))
+
+(deftest tick-with-a-fresh-matching-entry-does-nothing-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ (let [row (hash-row token)]
+ (token-check/tick!)
+ (is (= 1 @calls))
+ (is (= row (hash-row token)))
+ (is (=? {:valid true :status "ok"} (premium-features/token-status))))))))
+
+(def ^:private audit-app-status {:valid true :status "ok" :features ["audit-app"]})
+
+(defn- change-hash-row!
+ [token]
+ (t2/update! :model/PremiumFeaturesCache :token_hash (#'token-check/hash-token token)
+ {:token_status_hash (#'token-check/hash-token-status audit-app-status)}))
+
+(deftest clear-local-cache-with-a-matching-hash-row-does-not-fetch-test
+ (do-with-refresher-started!
+ (fn []
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ (dotimes [_ 3]
+ (premium-features/clear-local-cache!))
+ (is (= 1 @calls))
+ (is (not @@#'token-check/refresh-requested?))
+ (is (premium-features/has-feature? :sandboxes)))))))
+
+(deftest clear-local-cache-with-a-differing-hash-row-fetches-test
+ (do-with-refresher-started!
+ (fn []
+ (let [token (tu/random-token)
+ response (atom good-status)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [calls #(deref response)]
+ (token-check/refresh-now!)
+ (reset! response audit-app-status)
+ (change-hash-row! token)
+ (premium-features/clear-local-cache!)
+ (is (= 2 @calls))
+ (is (premium-features/has-feature? :audit-app))
+ (is (not (premium-features/has-feature? :sandboxes)))
+ (is (=? {:valid true :features ["audit-app"]} (premium-features/token-status)))
+ (testing "the hash row now matches the held status, so the next hint does nothing"
+ (premium-features/clear-local-cache!)
+ (is (= 2 @calls)))))))))
+
+(deftest clear-local-cache-with-no-held-status-fetches-test
+ (do-with-refresher-started!
+ (fn []
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (premium-features/clear-local-cache!)
+ (is (= 1 @calls))
+ (is (premium-features/has-feature? :sandboxes)))))))
+
+(deftest read-for-a-newly-configured-token-ticks-early-test
+ (do-with-refresher-started!
+ (fn []
+ (let [calls (atom 0)]
+ ;; the tick runs on the refresher thread, which does not see `with-dynamic-fn-redefs`
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ (with-redefs [token-check/http-fetch (fn [& _]
+ (swap! calls inc)
+ {:status 200 :body (json/encode good-status)})]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (is (not (premium-features/has-feature? :sandboxes))
+ "the read itself does not wait for the fetch")
+ (is (true? (u/poll {:thunk #(premium-features/has-feature? :sandboxes)
+ :done? true?
+ :timeout-ms 5000
+ :interval-ms 10})))
+ (is (= 1 @calls))
+ (testing "a token whose early tick already ran gets no second one"
+ (reset! @#'token-check/entry nil)
+ (premium-features/has-feature? :sandboxes)
+ (Thread/sleep 200)
+ (is (= 1 @calls)))))
+ (testing "a token whose refresh already failed waits for the scheduled tick"
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ ;; a tick, if one ran, would be on the refresher thread
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ (with-redefs [token-check/http-fetch (fn [& _]
+ (swap! calls inc)
+ (throw (java.net.ConnectException. "Connection refused")))]
+ (premium-features/refresh-at-startup!)
+ (reset! calls 0)
+ (premium-features/has-feature? :sandboxes)
+ (Thread/sleep 200)
+ (is (zero? @calls)))))))))
+
+(deftest clear-local-cache-whose-fetch-fails-does-not-throw-test
+ (do-with-refresher-started!
+ (fn []
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls #(throw (ex-info "network failure!" {}))]
+ (is (nil? (premium-features/clear-local-cache!)))
+ (is (pos? @calls)))))))
+
+(deftest clear-local-cache-before-the-refresher-starts-only-requests-a-refresh-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [calls (constantly good-status)]
+ (testing "with nothing held"
+ (premium-features/clear-local-cache!)
+ (is (zero? @calls) "a hint that arrives during initialization does not fetch")
+ (is @@#'token-check/refresh-requested?)
+ (token-check/tick!)
+ (is (= 1 @calls) "the first tick makes the refresh the hint asked for"))
+ (testing "with a differing hash row"
+ (change-hash-row! token)
+ (premium-features/clear-local-cache!)
+ (is (= 1 @calls))
+ (is @@#'token-check/refresh-requested?)
+ (token-check/tick!)
+ (is (= 2 @calls))
+ (is (premium-features/has-feature? :sandboxes)))))))
+
+(deftest tick-after-another-instance-changed-the-hash-fetches-test
+ (let [token (tu/random-token)
+ response (atom good-status)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [calls #(deref response)]
+ (token-check/refresh-now!)
+ (reset! response audit-app-status)
+ (change-hash-row! token)
+ (token-check/tick!)
+ (is (= 2 @calls))
+ (is (premium-features/has-feature? :audit-app))
+ (is (not (premium-features/has-feature? :sandboxes)))
+ (testing "the next tick sees the hash it now holds and does nothing"
+ (token-check/tick!)
+ (is (= 2 @calls)))))))
+
+(deftest tick-with-a-missing-hash-row-fetches-and-writes-it-back-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ (t2/delete! :model/PremiumFeaturesCache :token_hash (#'token-check/hash-token token))
+ (token-check/tick!)
+ (is (= 2 @calls) "a missing row triggers one fetch")
+ (is (= (:status-hash @@#'token-check/entry) (:token_status_hash (hash-row token))))
+ (is (premium-features/has-feature? :sandboxes))
+ (testing "the row is back, so the next tick does nothing"
+ (token-check/tick!)
+ (is (= 2 @calls)))))))
+
+(deftest tick-after-the-refresh-interval-fetches-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ (testing "an entry younger than 11h, the shortest jittered interval, is not refreshed"
+ (age-entry! (t/minus (t/hours 11) (t/minutes 1)))
+ (token-check/tick!)
+ (is (= 1 @calls)))
+ (testing "an entry older than 12h, the longest jittered interval, is"
+ (age-entry! (t/minutes 62))
+ (token-check/tick!)
+ (is (= 2 @calls))
+ (is (premium-features/has-feature? :sandboxes)))
+ (testing "the refreshed entry is fresh again"
+ (token-check/tick!)
+ (is (= 2 @calls))))))
+
+(deftest tick-after-a-requested-refresh-fetches-once-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ (token-check/request-refresh!)
+ (is (= 1 @calls) "requesting a refresh does not fetch")
+ (token-check/tick!)
+ (is (= 2 @calls))
+ (testing "the request is cleared, so the next tick does nothing"
+ (token-check/tick!)
+ (is (= 2 @calls))))))
+
+(deftest tick-whose-fetch-fails-leaves-the-entry-alone-test
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!))
+ (let [held @@#'token-check/entry
+ row (hash-row token)]
+ (with-store [calls #(throw (ex-info "network failure!" {}))]
+ (token-check/request-refresh!)
+ (is (nil? (token-check/tick!)))
+ (is (pos? @calls))
+ (is (= held @@#'token-check/entry))
+ (is (= row (hash-row token)))
+ (is (premium-features/has-feature? :sandboxes)))
+ (testing "the request stands, so the next tick tries again"
+ (with-store [calls (constantly good-status)]
+ (token-check/tick!)
+ (is (= 1 @calls))
+ (token-check/tick!)
+ (is (= 1 @calls))))))))
+
+(deftest tick-whose-hash-read-fails-does-not-throw-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ (mt/with-dynamic-fn-redefs [premium-features.db/token-status-cache (fn [_] (throw (ex-info "app DB down" {})))]
+ (is (nil? (token-check/tick!))))
+ (is (= 1 @calls))
+ (is (premium-features/has-feature? :sandboxes)))))
+
+(deftest tick-reads-the-app-db-at-most-once-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (token-check/refresh-now!)
+ ;; the first read may run the settings cache's own once-a-minute staleness check
+ (token-check/tick!)
+ (t2/with-call-count [call-count]
+ (token-check/tick!)
+ (is (= 1 (call-count)) "a tick with nothing due reads the hash row and nothing else"))
+ (t2/with-call-count [call-count]
+ (is (premium-features/has-feature? :sandboxes))
+ (is (zero? (call-count)) "a feature check reads nothing"))
+ (is (= 1 @calls)))))
+
+(defn- fetch-log-count
+ [messages]
+ (count (filter #(re-find #"Checking with the MetaStore" (:message %)) messages)))
+
+(deftest startup-refresh-on-a-healthy-store-fetches-once-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls (constantly good-status)]
+ (mt/with-log-messages-for-level [messages :info]
+ (is (nil? (premium-features/refresh-at-startup!)))
+ (is (= 1 (fetch-log-count (messages)))))
+ (is (= 1 @calls))
+ (is (premium-features/has-feature? :sandboxes)))))
+
+(deftest startup-refresh-on-an-unreachable-store-tries-once-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [calls #(throw (java.net.SocketTimeoutException. "Read timed out"))]
+ (is (nil? (premium-features/refresh-at-startup!)) "the failure does not escape")
+ (is (= 1 @calls) "the attempt is not retried")
+ (is (not (premium-features/has-feature? :sandboxes)))
+ (is (=? {:valid false :canonical? false} (premium-features/token-status))))
+ (testing "a later tick picks up features once the store returns"
+ (with-store [calls (constantly good-status)]
+ (token-check/tick!)
+ (is (= 1 @calls))
+ (is (premium-features/has-feature? :sandboxes))))))
+
+(defn- refresher-threads []
+ (filter #(= "premium-features-token-refresher" (.getName ^Thread %)) (keys (Thread/getAllStackTraces))))
+
+(deftest refresher-ticks-until-stopped-test
+ (let [ticks (atom 0)]
+ ;; the executor's thread does not inherit dynamic bindings
+ #_{:clj-kondo/ignore [:metabase/prefer-with-dynamic-fn-redefs]}
+ (with-redefs [token-check/tick! (fn []
+ (when (odd? (swap! ticks inc))
+ (throw (ex-info "tick failure!" {}))))]
(try
- (dotimes [_ 3] (token-check/check-token checker token))
- (is (= 3 @call-count) "non-canonical results are not cached")
- (is (=? {:valid false
- :canonical? false
- :error-details "Refusing to cache a non-authoritative token status"}
- (token-check/check-token checker token)))
+ (is (true? (premium-features/start-refresher! 1)))
+ (is (false? (premium-features/start-refresher! 1)) "a second start leaves the running refresher alone")
+ (is (u/poll {:thunk #(deref ticks) :done? #(>= % 4) :interval-ms 1})
+ "it keeps ticking after a tick throws")
+ (is (= 1 (count (refresher-threads))))
(finally
- (token-check/-clear-cache! checker))))))
-
-(deftest token-features-honors-valid-test
- (testing "*token-features* returns no features when the token status is not valid, even if features are present"
- (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
- (mt/with-dynamic-fn-redefs [token-check/check-token (fn [_token] {:valid false
- :canonical? true
- :status "Token is expired."
- :features ["sso"]})]
- (is (= #{} (token-check/*token-features*)))
- (is (false? (token-check/has-feature? :sso)))
- (is (false? (token-check/canonically-has-feature? :sso)))))))
-
-(deftest fetch-token-does-not-call-db-when-cached
- (testing "No DB calls are made when checking token status if the status is in local cache"
- (let [token (tu/random-token)
- checker (fresh-checker)
- _ (token-check/check-token checker token)
- ;; The local cache has a 5s TTL, so repeated checks within that window should not hit the DB.
- call-counts (repeatedly 3 (fn []
- (t2/with-call-count [call-count]
- (token-check/check-token checker token)
- (call-count))))]
- ;; At least some of these should be zero (served from local in-memory cache)
- (is (some zero? call-counts)))))
-
-(deftest token-checker-test
- (let [token (tu/random-token)
- behavior (atom :success)
- call-count (atom 0)
- good-response {:valid true
- :status "fake"
- :features ["feature1" "feature2"]
- :canonical? true}
- token-response (fn [_token]
- (swap! call-count inc)
- (case @behavior
- :success good-response
- :error (throw (ex-info "network issues!" {:ka :boom}))))
- checker (token-check/make-checker
- {:base (reify token-check/TokenChecker
- (-check-token [_ token]
- (token-response token))
- (-clear-cache! [_])
- (-clear-local-cache! [_]))
- :circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]
- :delay-ms 50
- :success-threshold 1}
- :timeout-ms 100
- :local-ttl (t/millis 50)
- :soft-ttl (t/millis 200)
- :hard-ttl (t/seconds 1)})]
- (testing "when no information is present, hits the underlying"
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count)))
- (testing "hitting it again in the same timeperiod does not hit the underlying checker"
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count)))
- (testing "While it errors, we have circuit breaking"
- (reset! behavior :error)
- (Thread/sleep 200)
- (dotimes [_ 1000] (token-check/-check-token checker token))
- (is (< @call-count 50)))
- (testing "When connectivity is restored we get successful token checks"
- (reset! behavior :success)
- (Thread/sleep 100) ;; wait for circuit breaker to open back up (also outlasts the 50ms negative cache)
- (is (= good-response (token-check/-check-token checker token)))))
- (testing "App db not setup yet does not invoke the circuit breaker (#65294), and the refusal is not cached past setup"
- (let [token (tu/random-token)
- call-count (atom 0)
- good-response {:valid true
- :status "fake"
- :features ["feature1" "feature2"]
- :canonical? true}
- checker (token-check/make-checker
- {:base (reify token-check/TokenChecker
- (-check-token [_ _token]
- (swap! call-count inc)
- good-response)
- (-clear-cache! [_])
- (-clear-local-cache! [_]))
- :circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]
- :delay-ms 5000
- :success-threshold 1}
- :timeout-ms 100
- :local-ttl (t/seconds 10)
- :soft-ttl (t/millis 200)
- :hard-ttl (t/seconds 1)})]
- (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly false)]
- (is (= {:valid false
- :canonical? false
- :status "Unable to validate token"
- :error-details "Metabase DB is not yet set up"}
- (token-check/-check-token checker token)))
- (dotimes [_ 50] (token-check/-check-token checker token))
- (is (= {:valid false
- :canonical? false
- :status "Unable to validate token"
- :error-details "Metabase DB is not yet set up"}
- (token-check/-check-token checker token))))
- (testing "When the db-is-set-up? we are not blocked by the circuit breaker, and the first check reaches the store"
+ (premium-features/stop-refresher!)))
+ (is (u/poll {:thunk refresher-threads :done? empty? :interval-ms 1}) "stopping leaves no thread behind")
+ (testing "it can be started again after a stop"
(try
- (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly true)]
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count)))
+ (is (true? (premium-features/start-refresher! 1)))
(finally
- (token-check/-clear-cache! checker)))))))
-
-(defn- age-cache-entry!
- "Rewrite the `updated_at` timestamp for `token` in the cache table to be `age-ms` milliseconds in the past.
- Also ages the local cache entry if a `local-cache` atom is provided."
- ([token age-ms]
- (age-cache-entry! token age-ms nil))
- ([token age-ms local-cache]
- (let [token-hash (#'token-check/hash-token token)
- new-ts (t/minus (t/offset-date-time) (t/millis age-ms))]
- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash {:updated_at new-ts})
- (when local-cache
- (swap! local-cache update token-hash assoc :updated-at (t/minus (t/instant) (t/millis age-ms)))))))
-
-(deftest e2e-test
- (testing "full stack: HTTP fetch → DB-hash-aware cache → error handling"
- (let [token (tu/random-token)
- call-count (atom 0)
- good-body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"
- good-resp {:valid true, :status "fake", :features ["fake" "features"], :canonical? true}
- local-cache (atom {})
- ;; no circuit breaker — it carries state between runs and causes flakes
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/millis 50)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)
- :db-hash-local-cache local-cache}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- {:status 200 :body good-body})]
- (is (= good-resp (token-check/check-token checker token)))
- (is (= 1 @call-count)))
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- (throw (ex-info "network failure!" {})))]
- (testing "Doesn't hit the network inside of cache duration"
- ;; Wait for local-cached-token-checker TTL to expire, but DB-hash-aware cache is still fresh
- (Thread/sleep 60)
- (is (= good-resp (token-check/check-token checker token)))
- (is (= 1 @call-count)))
- (testing "Stale cache (past soft TTL, within hard TTL) still serves cached result"
- (age-cache-entry! token (+ (u/hours->ms 12) 1000) local-cache)
- (Thread/sleep 60) ;; expire local-cached-token-checker
- (let [refresh-done (promise)]
- (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]
- (is (= good-resp (token-check/check-token checker token))))
- ;; async refresh was attempted (and failed), but stale value was returned
- (is (true? (deref refresh-done 5000 :timeout))))
- (is (= 2 @call-count)))
- (testing "Expired cache (past hard TTL) surfaces the error"
- (age-cache-entry! token (+ (u/hours->ms 36) 1000) local-cache)
- (Thread/sleep 60) ;; expire local-cached-token-checker
- (is (= {:valid false
- :canonical? false
- :status "Unable to validate token"
- :error-details "network failure!"}
- (token-check/check-token checker token)))))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest license-server-fields-we-do-not-read-test
- (testing "a token status response carrying fields we don't read still validates the token"
- (let [token (tu/random-token)
- body {:valid true
- :status "ok"
- :features ["sso-jwt"]
- :plan-alias "pro-self-hosted"
- :new-field "added by the license server"
- :store-users [{:email "[email protected]" :id 42 :first-name "Owner" :last-name "Person"}]
- :quotas [{:hosting-feature "metabase-ai-tokens"
- :soft-limit 100
- :usage 5
- :locked false
- :updated-at "2026-09-01T00:00:00Z"
- :quota-type "monthly"}]}
- checker (fresh-checker)]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _] {:status 200 :body (json/encode body)})]
- (is (= (assoc body :canonical? true)
- (token-check/check-token checker token))))
- (finally
- (token-check/-clear-cache! checker))))))
+ (premium-features/stop-refresher!)))))))
+
+(deftest quotas-and-meters-reflect-a-fresh-fetch-test
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (let [usage (atom 1)]
+ (with-store [calls (fn [] {:valid true
+ :status "ok"
+ :quotas [{:hosting-feature "metabase-ai-tokens" :usage @usage}]
+ :meters {:metabase-ai-tokens {:meter-value @usage}}})]
+ (is (=? [{:usage 1}] (premium-features/quotas)))
+ (swap! usage inc)
+ (is (=? {:metabase-ai-tokens {:meter-value 2}} (premium-features/meters)))
+ (is (= 2 @calls))))))
+
+(deftest log-test
+ (let [token (tu/random-token)
+ print-token (apply str (concat (take 4 token) "..." (take-last 4 token)))]
+ (testing "Do not log the token (#18249)"
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [_ (constantly good-status)]
+ (mt/with-log-messages-for-level [messages :info]
+ (token-check/refresh-now!)
+ (let [logs (mapv :message (messages))]
+ (is (every? (complement #(re-find (re-pattern token) %)) logs))
+ (is (= 1 (count (filter #(re-find (re-pattern print-token) %) logs)))))))))))
(deftest ^:parallel extract-locks-test
(testing "empty :meters map yields empty result"
@@ -521,72 +650,7 @@
(#'token-check/extract-locks {:transform-basic-runs {:is-locked true}
:metabase-ai-tokens {:is-locked false}})))))
-(deftest do-refresh-writes-locked-meters-test
- (testing "do-refresh! mirrors :meters → :locked-meters setting on every successful refresh"
- (mt/with-temporary-setting-values [locked-meters {}]
- (let [token (tu/random-token)
- response (atom {:valid true :status "ok" :canonical? true})
- local-cache (atom {})
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/millis 50)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)
- :db-hash-local-cache local-cache}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch
- (fn [& _] {:status 200 :body (json/encode @response)})]
- (testing "successful response with :meters writes through"
- (reset! response {:valid true :status "ok"
- :meters {:transform-basic-runs {:is-locked true}
- :transform-advanced-runs {:is-locked false}}})
- (token-check/check-token checker token)
- (is (= {:transform-basic-runs true
- :transform-advanced-runs false}
- (premium-features/locked-meters))))
- (testing "successful response WITHOUT :meters leaves the setting untouched"
- (reset! response {:valid true :status "ok"})
- (token-check/-clear-cache! checker)
- (reset! local-cache {})
- (token-check/check-token checker token)
- (is (= {:transform-basic-runs true
- :transform-advanced-runs false}
- (premium-features/locked-meters))
- "Setting should retain previous value when response omits :meters"))
- (testing "successful response with empty :meters {} writes empty map (legitimate unlock)"
- (reset! response {:valid true :status "ok" :meters {}})
- (token-check/-clear-cache! checker)
- (reset! local-cache {})
- (token-check/check-token checker token)
- (is (= {} (premium-features/locked-meters)))))
- (finally
- (token-check/-clear-cache! checker)))))))
-
-(deftest do-refresh-failure-leaves-locked-meters-untouched-test
- (testing "Outage / 5xx / circuit breaker open → :locked-meters survives unchanged.
- (Critical correctness property: a network blip MUST NOT accidentally unlock.)"
- (mt/with-temporary-setting-values [locked-meters {:transform-basic-runs true}]
- (let [token (tu/random-token)
- local-cache (atom {})
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:local-ttl (t/millis 50)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)
- :db-hash-local-cache local-cache}))]
- (try
- (mt/with-dynamic-fn-redefs [token-check/http-fetch
- (fn [& _] (throw (ex-info "network failure!" {})))]
- (token-check/check-token checker token)
- (is (= {:transform-basic-runs true}
- (premium-features/locked-meters))
- "Failed refresh must not touch :locked-meters"))
- (finally
- (token-check/-clear-cache! checker)))))))
-
-(deftest token-status-setting-test
- (testing "If a `premium-embedding-token` has been set, the `token-status` setting should return the response
- from the store.metabase.com endpoint for that token."
- (is (= {:valid false, :status "Token does not exist.", :canonical? true}
- (token-check/check-token (fresh-checker) (tu/random-token)))))
+(deftest token-status-without-token-test
(testing "If premium-embedding-token is nil, the token-status setting should also be nil."
(mt/with-temporary-setting-values [premium-embedding-token nil]
(is (nil? (premium-features/token-status))))))
@@ -720,17 +784,12 @@
(let [;; This is a fake airgap token format (starts with "airgap_")
airgap-token "airgap_eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ90faketoken"
request-made (atom false)]
- (mt/with-dynamic-fn-redefs [token-check/check-token
- (constantly {:valid true
- :status "fake"
- :features ["test" "fixture"]
- :trial false})]
- (mt/with-temporary-raw-setting-values [premium-embedding-token airgap-token]
- (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts]
- (reset! request-made true)
- {:status 200 :body "{}"})]
- (token-check/send-metering-events!)
- (is (false? @request-made) "No request should be made for airgap tokens")))))))
+ (mt/with-temporary-raw-setting-values [premium-embedding-token airgap-token]
+ (mt/with-dynamic-fn-redefs [http/post (fn [_url _opts]
+ (reset! request-made true)
+ {:status 200 :body "{}"})]
+ (token-check/send-metering-events!)
+ (is (false? @request-made) "No request should be made for airgap tokens"))))))
(deftest metering-stats-test
(testing "metering-stats returns expected keys"
@@ -775,380 +834,81 @@
:let [plan (-> (t2/query (into [(str "EXPLAIN " sql)] params)) first :plan)]]
(is (re-find #"IDX_QUERY_EXECUTION_STARTED_AT: STARTED_AT" plan)))))))
-;;; ------------------------------------------------ db-hash-aware-token-checker ------------------------------------------------
-
-(defn- make-db-hash-aware-checker
- "Helper: wraps a base token-response fn with DB-hash-aware caching. Returns the checker."
- [token-response-fn {:keys [local-ttl soft-ttl hard-ttl]
- :or {local-ttl (t/millis 50)}}]
- (binding [token-check/*customize-checker* true]
- (token-check/make-checker
- {:base (reify token-check/TokenChecker
- (-check-token [_ token] (token-response-fn token))
- (-clear-cache! [_])
- (-clear-local-cache! [_]))
- :local-ttl local-ttl
- :soft-ttl soft-ttl
- :hard-ttl hard-ttl})))
-
-(deftest clear-local-cache-test
- (testing "clear-local-cache! drops in-process memoization but leaves the shared DB row alone"
- (let [token (tu/random-token)
- token-hash (#'token-check/hash-token token)
- call-count (atom 0)
- deletes (atom 0)
- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- local-cache (atom {})
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker
- {:base (reify token-check/TokenChecker
- (-check-token [_ _token]
- (swap! call-count inc)
- good-response)
- (-clear-cache! [_])
- (-clear-local-cache! [_]))
- ;; long enough that nothing expires during the test
- :local-ttl (t/hours 1)
- :soft-ttl (t/hours 12)
- :hard-ttl (t/hours 36)
- :db-hash-local-cache local-cache}))]
- (try
- (is (= good-response (token-check/check-token checker token)))
- (is (= 1 @call-count))
- (mt/with-dynamic-fn-redefs [premium-features.db/delete-token-status-cache! (fn [] (swap! deletes inc))]
- (token-check/-clear-local-cache! checker))
- (is (zero? @deletes) "the shared DB row is never deleted")
- (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))
- (is (contains? @local-cache token-hash) "the DB-hash-validated local entry survives")
- (testing "the next check re-validates against the DB hash and, since it matches, skips the MetaStore"
- (is (= good-response (token-check/check-token checker token)))
- (is (= 1 @call-count)))
- (testing "but a refresh done by another instance (DB hash changed) is picked up"
- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash {:token_status_hash "refreshed-elsewhere"})
- (testing "not while the in-process memo still holds"
- (is (= good-response (token-check/check-token checker token)))
- (is (= 1 @call-count)))
- (token-check/-clear-local-cache! checker)
- (is (= good-response (token-check/check-token checker token)))
- (is (= 2 @call-count)))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest db-hash-aware-token-checker-fresh-hit-test
- (testing "Fresh cache entry (< soft-ttl) is returned without hitting the underlying checker"
- (let [call-count (atom 0)
- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- checker (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count inc)
- good-response)
- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
- token (tu/random-token)]
- (try
- ;; First call: cache miss → delegate
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count))
- ;; Second call: fresh cache hit → no delegate
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest db-hash-aware-token-checker-stale-async-refresh-test
- (testing "Stale entry (soft..hard) returns cached value and triggers async refresh"
- (let [call-count (atom 0)
- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- updated-response {:valid true :status "OK" :features ["sandboxes" "audit-app"] :canonical? true}
- response-atom (atom good-response)
- refresh-done (promise)
- checker (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count inc)
- @response-atom)
- ;; soft=1ms so it's immediately stale, hard=10s so not expired
- {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/seconds 10)})
- token (tu/random-token)]
- (try
- ;; First call: cache miss → delegate
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count))
- ;; Let the entry become stale (both local and DB)
- (Thread/sleep 10)
- ;; Switch to updated response for the async refresh
- (reset! response-atom updated-response)
- ;; Second call: stale → returns old cached value, kicks off async refresh
- (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]
- (is (= good-response (token-check/-check-token checker token)))
- ;; bounded deref — unbounded hangs the whole run if the refresh never fires
- (is (true? (deref refresh-done 5000 :timeout))))
- ;; Expire local cache
- (Thread/sleep 10)
- ;; Third call: should now see the refreshed value
- (is (= updated-response (token-check/-check-token checker token)))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest db-hash-aware-token-checker-expired-sync-test
- (testing "Expired entry (> hard-ttl) delegates synchronously"
- (let [call-count (atom 0)
- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- checker (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count inc)
- good-response)
- ;; Both TTLs = 1ms so entry expires immediately
- {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})
- token (tu/random-token)]
- (try
- ;; First call: miss
- (token-check/-check-token checker token)
- (is (= 1 @call-count))
- (Thread/sleep 5)
- ;; Second call: expired → synchronous delegate
- (token-check/-check-token checker token)
- (is (= 2 @call-count))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest db-hash-aware-token-checker-multi-token-test
- (testing "Different tokens are cached independently"
- (let [call-count (atom 0)
- checker (make-db-hash-aware-checker
- (fn [token]
- (swap! call-count inc)
- {:valid true :status "OK" :features [(str "feat-" token)] :canonical? true})
- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
- token-a (tu/random-token)
- token-b (tu/random-token)]
- (try
- (let [result-a (token-check/-check-token checker token-a)]
- (is (= [(str "feat-" token-a)] (:features result-a))))
- (let [result-b (token-check/-check-token checker token-b)]
- (is (= [(str "feat-" token-b)] (:features result-b))))
- (is (= 2 @call-count))
- ;; Both cached now
- (token-check/-check-token checker token-a)
- (token-check/-check-token checker token-b)
- (is (= 2 @call-count))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest db-hash-aware-token-checker-clear-cache-test
- (testing "clear-cache! removes both local and DB cache"
- (let [checker (make-db-hash-aware-checker
- (fn [_token]
- {:valid true :status "OK" :features ["sandboxes"] :canonical? true})
- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
- token (tu/random-token)]
- (token-check/-check-token checker token)
- (let [token-hash (#'token-check/hash-token token)]
- (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))
- (token-check/-clear-cache! checker)
- (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))))))
-
-(deftest db-hash-aware-token-checker-invalid-cached-test
- (testing "Invalid token responses are also cached in table to avoid hammering MetaStore"
- (let [call-count (atom 0)
- checker (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count inc)
- {:valid false :status "Token does not exist." :canonical? true})
- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
- token (tu/random-token)]
- (try
- (is (= {:valid false :status "Token does not exist." :canonical? true}
- (token-check/-check-token checker token)))
- (is (= 1 @call-count))
- ;; Second call should use cache
- (is (= {:valid false :status "Token does not exist." :canonical? true}
- (token-check/-check-token checker token)))
- (is (= 1 @call-count))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest db-hash-aware-token-checker-cross-instance-sync-test
- (testing "When instance A refreshes and gets new features, instance B detects the hash mismatch and re-fetches"
- (let [call-count-a (atom 0)
- call-count-b (atom 0)
- old-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- new-response {:valid true :status "OK" :features ["sandboxes" "audit-app"] :canonical? true}
- response-a (atom old-response)
- response-b (atom old-response)
- checker-a (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count-a inc)
- @response-a)
- {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})
- checker-b (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count-b inc)
- @response-b)
- {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})
- token (tu/random-token)]
- (try
- ;; Both instances populate their caches with old-response
- (is (= old-response (token-check/-check-token checker-a token)))
- (is (= old-response (token-check/-check-token checker-b token)))
- (is (= 1 @call-count-a))
- (is (= 1 @call-count-b))
- ;; Instance A gets new features and refreshes (expired TTL forces sync refresh)
- (reset! response-a new-response)
- (Thread/sleep 5) ;; expire checker-a's local + DB TTLs
- (is (= new-response (token-check/-check-token checker-a token)))
- (is (= 2 @call-count-a))
- ;; Instance B has a long TTL, but the DB hash now differs from its local hash
- ;; So it should detect the mismatch and do a synchronous re-fetch
- (reset! response-b new-response)
- (Thread/sleep 5) ;; expire local-cached-token-checker's 1ms TTL
- (is (= new-response (token-check/-check-token checker-b token)))
- (is (= 2 @call-count-b))
- (finally
- (token-check/-clear-cache! checker-a)
- (token-check/-clear-cache! checker-b))))))
-
-(deftest db-hash-aware-token-checker-exception-propagation-test
- (testing "When delegate throws on expired/missing, exception propagates through error-catching"
- (let [checker (make-db-hash-aware-checker
- (fn [_token]
- (throw (ex-info "MetaStore unreachable" {})))
- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
- token (tu/random-token)]
- ;; error-catching wraps it, so we get the error-details response
- (is (= {:valid false
- :canonical? false
- :status "Unable to validate token"
- :error-details "MetaStore unreachable"}
- (token-check/-check-token checker token))))))
-
-(deftest db-hash-aware-token-checker-tamper-resistance-test
- (testing "Tampering with the DB hash triggers a re-fetch from MetaStore, not a cache hit"
- (let [call-count (atom 0)
- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- checker (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count inc)
- good-response)
- {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})
- token (tu/random-token)
- token-hash (#'token-check/hash-token token)]
- (try
- ;; Populate the cache
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count))
- ;; Tamper with the DB hash — simulate someone modifying the row
- (t2/update! :model/PremiumFeaturesCache :token_hash token-hash
- {:token_status_hash "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"})
- ;; Expire local-cached-token-checker
- (Thread/sleep 5)
- ;; Next check should detect hash mismatch and re-fetch from MetaStore
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 2 @call-count))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest startup-clears-token-cache-test
- (testing "The startup hook clears the token cache table so the first check after boot hits MetaStore"
- (let [call-count (atom 0)
- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- checker (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count inc)
- good-response)
- {:soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})
- token (tu/random-token)]
- (try
- ;; Populate the cache
- (token-check/-check-token checker token)
- (is (= 1 @call-count))
- (let [token-hash (#'token-check/hash-token token)]
- (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))
- ;; Run the startup hook (clears the global token-checker's cache, including the table)
- (startup/def-startup-logic! :metabase.premium-features.task.clear-token-cache/clear-token-cache)
- ;; Table should be cleared
- (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash))))
- (finally
- (token-check/-clear-cache! checker))))))
-
-(deftest db-hash-aware-token-checker-db-not-set-up-test
- (testing "When DB is not set up, delegates directly to inner checker without touching the DB"
- (let [call-count (atom 0)
- good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
- checker (make-db-hash-aware-checker
- (fn [_token]
- (swap! call-count inc)
- good-response)
- {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
- token (tu/random-token)
- bomb (fn [& _] (throw (ex-info "DB should not be touched" {})))]
- (mt/with-dynamic-fn-redefs [mdb/db-is-set-up? (constantly false)
- token-check/read-cache-from-db bomb
- token-check/write-cache-to-db! bomb
- token-check/clear-db-cache! bomb]
- (is (= good-response (token-check/-check-token checker token)))
- (is (= 1 @call-count) "inner checker was called exactly once")
- ;; clear-cache! should also skip DB without error
- (token-check/-clear-cache! checker)))))
+(deftest license-server-fields-we-do-not-read-test
+ (testing "a token status response carrying fields we don't read still validates the token"
+ (let [body {:valid true
+ :status "ok"
+ :features ["sso-jwt"]
+ :plan-alias "pro-self-hosted"
+ :new-field "added by the license server"
+ :store-users [{:email "[email protected]" :id 42 :first-name "Owner" :last-name "Person"}]
+ :quotas [{:hosting-feature "metabase-ai-tokens"
+ :soft-limit 100
+ :usage 5
+ :locked false
+ :updated-at "2026-09-01T00:00:00Z"
+ :quota-type "monthly"}]}]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
+ (with-store [_ (constantly body)]
+ (is (= (assoc body :canonical? true) (token-check/refresh-now!)))
+ (is (= #{"sso-jwt"} (token-check/*token-features*))))))))
;;; ------------------------------------------------ -set-premium-embedding-token! ------------------------------------------------
(deftest set-premium-embedding-token-canonical-invalid-test
(testing "When MetaStore says the token is invalid (canonical), throws 400 with MetaStore's message"
- (let [token (tu/random-token)]
- (mt/with-dynamic-fn-redefs [token-check/check-token (constantly {:valid false
- :status "Token expired"
- :error-details "Expired 2024-01-01"
- :canonical? true})]
- (let [e (try (token-check/-set-premium-embedding-token! token)
- nil
- (catch Exception e e))]
- (is (some? e))
- (is (= "Token expired" (ex-message e)))
- (is (= 400 (:status-code (ex-data e))))
- (is (= "Expired 2024-01-01" (:error-details (ex-data e)))))))))
-
-(deftest set-premium-embedding-token-non-canonical-failure-test
- (testing "When token validation fails for non-canonical reasons (network etc), throws 503"
- (let [token (tu/random-token)]
- (mt/with-dynamic-fn-redefs [token-check/check-token (constantly {:valid false
- :status "Unable to validate token"
- :error-details "Connection refused"})]
- (let [e (try (token-check/-set-premium-embedding-token! token)
- nil
- (catch Exception e e))]
- (is (some? e))
- (is (= "Unable to validate token" (ex-message e)))
- (is (= 503 (:status-code (ex-data e))))
- (is (= "Connection refused" (:error-details (ex-data e)))))))))
+ (let [old-token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token old-token]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!))
+ (with-store [_ (constantly {:status 403 :body (json/encode {:valid false
+ :status "Token expired"
+ :error-details "Expired 2024-01-01"})})]
+ (let [e (try (token-check/-set-premium-embedding-token! (tu/random-token))
+ nil
+ (catch Exception e e))]
+ (is (= "Token expired" (ex-message e)))
+ (is (= 400 (:status-code (ex-data e))))
+ (is (= "Expired 2024-01-01" (:error-details (ex-data e)))))
+ (testing "the rejected token is not stored, and the old token's features keep serving"
+ (is (= old-token (premium-features/premium-embedding-token)))
+ (is (= #{"sandboxes"} (token-check/*token-features*)))))))))
+
+(deftest set-premium-embedding-token-failure-test
+ (testing "When the store cannot be reached, throws 503"
+ (with-store [_ #(throw (ex-info "Connection refused" {}))]
+ (let [e (try (token-check/-set-premium-embedding-token! (tu/random-token))
+ nil
+ (catch Exception e e))]
+ (is (= "Unable to validate token" (ex-message e)))
+ (is (= 503 (:status-code (ex-data e))))
+ (is (= "Connection refused" (:error-details (ex-data e))))))))
(deftest set-premium-embedding-token-valid-test
- (testing "When token is valid, no exception is thrown and setting is persisted"
+ (testing "When the token is valid, it is persisted and its features serve at once"
(let [token (tu/random-token)]
- (mt/with-dynamic-fn-redefs [token-check/check-token (constantly {:valid true
- :status "OK"
- :features ["test"]})]
- (mt/with-temporary-setting-values [premium-embedding-token nil]
+ (mt/with-temporary-setting-values [premium-embedding-token nil]
+ (with-store [calls (constantly good-status)]
(token-check/-set-premium-embedding-token! token)
- (is (= token (premium-features/premium-embedding-token))))))))
-
-(deftest set-premium-embedding-token-busts-stale-cache-test
- (testing "re-entering a token after renewal revalidates with the store instead of a cached expired verdict (EMB-2341)"
- (let [token (tu/random-token)
- expired (json/encode {:valid false :status "Token is expired."})
- renewed (json/encode {:valid true :status "Token is valid." :features ["sso"]})
- state (atom :expired)]
- (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
- (case @state
- :expired {:status 403 :body expired}
- :renewed {:status 200 :body renewed}))]
- (try
- ;; prime the cache with the expired verdict
- (is (=? {:valid false :canonical? true} (token-check/check-token token)))
+ (is (= token (premium-features/premium-embedding-token)))
+ (is (= #{"sandboxes"} (token-check/*token-features*)))
+ (is (some? (hash-row token)))
+ (is (= 1 @calls)))))))
+
+(deftest set-premium-embedding-token-revalidates-test
+ (testing "re-entering a token after renewal revalidates with the store instead of the held expired verdict (EMB-2341)"
+ (let [token (tu/random-token)
+ state (atom :expired)]
+ (mt/with-temporary-setting-values [premium-embedding-token nil]
+ (with-store [_ #(case @state
+ :expired {:status 403 :body (json/encode {:valid false :status "Token is expired."})}
+ :renewed {:valid true :status "Token is valid." :features ["sso"]})]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (is (=? {:valid false :canonical? true} (token-check/refresh-now!))))
(reset! state :renewed)
- (mt/with-temporary-setting-values [premium-embedding-token nil]
- (token-check/-set-premium-embedding-token! token)
- (is (= token (premium-features/premium-embedding-token))))
- (finally
- (token-check/clear-cache!)))))))
+ (token-check/-set-premium-embedding-token! token)
+ (is (= token (premium-features/premium-embedding-token)))
+ (is (= #{"sso"} (token-check/*token-features*))))))))
(deftest invalid-token-error-response-test
(testing "an invalid token's error response carries its `:error-details`, and nothing else from the ex-data"
@@ -1158,3 +918,38 @@
:error-details "Token should be 64 hexadecimal characters."}
(api-error/response-data data)))
(is (= 400 (:status-code data))))))
+
+(deftest refresh-for-a-replaced-token-does-not-overwrite-the-new-one-test
+ (testing "a refresh for the old token that finishes after the setter stored a new one leaves the new status held"
+ (let [old-token (tu/random-token)
+ new-token (tu/random-token)
+ fetching (promise)
+ release (promise)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token old-token]
+ (mt/with-dynamic-fn-redefs [token-check/http-fetch
+ (fn [_base-url token _site-uuid]
+ (if (= token old-token)
+ (do (deliver fetching true)
+ @release
+ {:status 200 :body (json/encode {:valid true :status "ok" :features ["old"]})})
+ {:status 200 :body (json/encode {:valid true :status "ok" :features ["new"]})}))]
+ (let [old-refresh (future (token-check/refresh-now!))]
+ (is (true? (deref fetching 5000 :timeout)))
+ (token-check/-set-premium-embedding-token! new-token)
+ (is (= #{"new"} (token-check/*token-features*)))
+ (deliver release true)
+ (is (=? {:features ["old"]} (deref old-refresh 5000 :timeout)))
+ (is (= #{"new"} (token-check/*token-features*)))))))))
+
+(deftest no-app-db-with-a-token-configured-test
+ (testing "with a token configured and no app DB, a feature check reads only memory and does not throw"
+ (let [token (tu/random-token)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token token]
+ (with-store [_ (constantly good-status)]
+ (token-check/refresh-now!)))
+ (mt/with-dynamic-fn-redefs [premium-features.settings/premium-embedding-token (constantly token)
+ mdb/db-is-set-up? (constantly false)]
+ (is (= #{"sandboxes"} (token-check/*token-features*)))
+ (reset! @#'token-check/entry nil)
+ (is (= #{} (token-check/*token-features*)))
+ (is (=? {:valid false :canonical? false} (premium-features/token-status)))))))
diff --git a/test/metabase/server/middleware/premium_features_cache_test.clj b/test/metabase/server/middleware/premium_features_cache_test.clj
index fe43ccc9bb8..ddb56ae8347 100644
--- a/test/metabase/server/middleware/premium_features_cache_test.clj
+++ b/test/metabase/server/middleware/premium_features_cache_test.clj
@@ -2,8 +2,11 @@
(:require
[clojure.test :refer :all]
[metabase.premium-features.core :as premium-features]
+ [metabase.premium-features.test-util :as premium-features.tu]
+ [metabase.premium-features.token-check :as token-check]
[metabase.server.middleware.premium-features-cache :as mw.pf-cache]
- [metabase.test :as mt])
+ [metabase.test :as mt]
+ [metabase.util.json :as json])
(:import
(java.time Duration Instant)
(java.util.concurrent CountDownLatch)))
@@ -110,3 +113,30 @@
(.countDown start)
(run! deref futures)
(is (= 1 @calls))))))))
+
+(deftest newer-cookie-refreshes-token-status-test
+ (testing "a newer cookie makes an instance that holds no status fetch it from the store"
+ (let [entry (var-get #'token-check/entry)
+ saved @entry
+ calls (atom 0)
+ ;; an hour between ticks, so none runs during the test
+ started? (premium-features/start-refresher! (* 60 60 1000))]
+ (reset! entry nil)
+ (try
+ (with-redefs [mw.pf-cache/last-known-invalidation (atom nil)]
+ (mt/with-temporary-raw-setting-values [premium-embedding-token (premium-features.tu/random-token)]
+ (mt/with-dynamic-fn-redefs [token-check/http-fetch (fn [& _]
+ (swap! calls inc)
+ {:status 200
+ :body (json/encode {:valid true
+ :status "ok"
+ :features ["sandboxes"]})})]
+ ;; not a feature read: that would tick early on the refresher thread
+ (is (nil? @entry))
+ (run-middleware (request-with-cookie (str (Instant/now))) {:status 200})
+ (is (= 1 @calls))
+ (is (true? (premium-features/has-feature? :sandboxes))))))
+ (finally
+ (when started?
+ (premium-features/stop-refresher!))
+ (reset! entry saved))))))
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index f18ac746ea0..55bb01deca2 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -755,12 +755,13 @@
The token-value binding will contain the random token that was set."
[[token-value] & body]
- `(let [~token-value (premium-features.test-util/random-token)]
- (with-redefs [metabase.premium-features.token-check/check-token
- (constantly {:valid true
- :status "fake"
- :features ["test" "fixture"]
- :trial false})]
+ `(let [~token-value (premium-features.test-util/random-token)
+ status# {:valid true
+ :status "fake"
+ :features ["test" "fixture"]
+ :trial false}]
+ (with-redefs [metabase.premium-features.token-check/current-status (constantly status#)
+ metabase.premium-features.token-check/fetch-status (constantly status#)]
(with-temporary-raw-setting-values [:premium-embedding-token ~token-value]
~@body))))