Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Implement DB-level caching of premium features (#69241)

John Swanson34837bc9bbbe
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 77a5fa49e65..a1bad371174 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1025,6 +1025,7 @@
events
internal-stats
settings
+ startup
task
util}}
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 1089be59da4..c88c7496266 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -97,6 +97,7 @@
"PermissionsGroupMembership"
"PermissionsRevision"
"PersistedInfo"
+ "PremiumFeaturesCache"
"Pulse"
"PulseCard"
"PulseChannel"
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 899db5d9d41..5dd17327901 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -72,6 +72,7 @@
:model/PermissionsGroupMembership
:model/PermissionsRevision
:model/PersistedInfo
+ :model/PremiumFeaturesCache
:model/Pulse
:model/PulseCard
:model/PulseChannel
diff --git a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsx
index bd142eb3145..7fc4466e505 100644
--- a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsx
@@ -7,6 +7,7 @@ import {
setupStoreEEBillingEndpoint,
setupStoreEECloudAddOnsEndpoint,
setupStoreEETieredMetabotAI,
+ setupTokenRefreshEndpoint,
} from "__support__/server-mocks";
import { renderWithProviders, screen } from "__support__/ui";
import {
@@ -103,6 +104,7 @@ const setup = async ({
simulate_http_get_error,
);
setupStoreEETieredMetabotAI(simulate_http_post_error);
+ setupTokenRefreshEndpoint();
renderWithProviders(<MetabotPurchasePage />, {
// Calling `setupPropertiesEndpoints` above is not enough; we also need to set `storeInitialState`:
diff --git a/frontend/src/metabase/api/index.ts b/frontend/src/metabase/api/index.ts
index af76ea8a9de..d2e6177e4b4 100644
--- a/frontend/src/metabase/api/index.ts
+++ b/frontend/src/metabase/api/index.ts
@@ -29,6 +29,7 @@ export * from "./notification";
export * from "./parameters";
export * from "./permission";
export * from "./persist";
+export * from "./premium-features";
export * from "./revision";
export * from "./search";
export * from "./segment";
diff --git a/frontend/src/metabase/api/premium-features.ts b/frontend/src/metabase/api/premium-features.ts
new file mode 100644
index 00000000000..6a7cbaa1872
--- /dev/null
+++ b/frontend/src/metabase/api/premium-features.ts
@@ -0,0 +1,19 @@
+import type { TokenStatus } from "metabase-types/api";
+
+import { Api } from "./api";
+import { invalidateTags, tag } from "./tags";
+
+export const premiumFeaturesApi = Api.injectEndpoints({
+ endpoints: (builder) => ({
+ refreshTokenStatus: builder.mutation<TokenStatus, void>({
+ query: () => ({
+ method: "POST",
+ url: "/api/premium-features/token/refresh",
+ }),
+ invalidatesTags: (_, error) =>
+ invalidateTags(error, [tag("session-properties")]),
+ }),
+ }),
+});
+
+export const { useRefreshTokenStatusMutation } = premiumFeaturesApi;
diff --git a/frontend/src/metabase/api/utils/use-token-refresh.ts b/frontend/src/metabase/api/utils/use-token-refresh.ts
index cff7742dd5c..01e30ffd043 100644
--- a/frontend/src/metabase/api/utils/use-token-refresh.ts
+++ b/frontend/src/metabase/api/utils/use-token-refresh.ts
@@ -1,6 +1,10 @@
-import { useEffect } from "react";
+import { useCallback, useEffect } from "react";
-import { Api, useGetSettingsQuery } from "metabase/api";
+import {
+ Api,
+ useGetSettingsQuery,
+ useRefreshTokenStatusMutation,
+} from "metabase/api";
import { useDispatch } from "metabase/lib/redux";
import type { TokenStatusFeature } from "metabase-types/api";
@@ -47,6 +51,18 @@ export function useTokenRefreshUntil(
/* in order to force this hook to re-run on every request, even if the response data is the same, we can't destructure only the data prop from this hook, as is the pattern in many components */
const res = useGetSettingsQuery();
const dispatch = useDispatch();
+ const [refreshTokenStatus] = useRefreshTokenStatusMutation();
+
+ const refreshToken = useCallback(async () => {
+ // Bust the server-side cache first; the mutation's invalidatesTags will
+ // also invalidate session-properties on success, but we do it in finally
+ // too so the UI updates even if the request fails.
+ try {
+ await refreshTokenStatus().unwrap();
+ } finally {
+ dispatch(Api.util.invalidateTags(["session-properties"]));
+ }
+ }, [dispatch, refreshTokenStatus]);
useEffect(() => {
if (skip) {
@@ -57,10 +73,8 @@ export function useTokenRefreshUntil(
const isTokenFeatureMissing = !tokenStatusFeatures?.includes(tokenFeature);
if (isTokenFeatureMissing) {
- const timeout = setTimeout(() => {
- dispatch(Api.util.invalidateTags(["session-properties"]));
- }, intervalMs);
+ const timeout = setTimeout(refreshToken, intervalMs);
return () => clearTimeout(timeout);
}
- }, [res, dispatch, tokenFeature, intervalMs, skip]);
+ }, [res, dispatch, tokenFeature, intervalMs, skip, refreshToken]);
}
diff --git a/frontend/test/__support__/server-mocks/premium-features.ts b/frontend/test/__support__/server-mocks/premium-features.ts
index 7e6760e50fc..9343e172d5a 100644
--- a/frontend/test/__support__/server-mocks/premium-features.ts
+++ b/frontend/test/__support__/server-mocks/premium-features.ts
@@ -26,6 +26,12 @@ export const setupTokenStatusEndpointEmpty = () => {
});
};
+export const setupTokenRefreshEndpoint = () => {
+ fetchMock.post("path:/api/premium-features/token/refresh", 200, {
+ name: "premium-token-refresh",
+ });
+};
+
export const setupTokenActivationEndpoint = ({
success,
status,
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index 3cbc52ef5ae..9e30e59a2ad 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -2986,6 +2986,35 @@ databaseChangeLog:
constraints:
nullable: true
+ - changeSet:
+ id: v59.2026-02-07T17:00:00
+ author: johnswanson
+ comment: Create premium_features_token_cache table for cross-instance token cache
+ changes:
+ - createTable:
+ tableName: premium_features_token_cache
+ remarks: Caches premium features token check results, shared across instances
+ columns:
+ - column:
+ name: token_hash
+ type: varchar(64)
+ remarks: SHA-256 hex hash of the premium token
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: token_status_hash
+ type: varchar(64)
+ remarks: SHA-256 hex hash of the JSON-encoded token status
+ constraints:
+ nullable: false
+ - column:
+ name: updated_at
+ type: ${timestamp_type}
+ remarks: When this cache entry was last refreshed
+ constraints:
+ nullable: false
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index d5e92c1d903..6597aaa8d63 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -73,6 +73,7 @@
:model/PermissionsGroupMembership metabase.permissions.models.permissions-group-membership
:model/PermissionsRevision metabase.permissions.models.permissions-revision
:model/PersistedInfo metabase.model-persistence.models.persisted-info
+ :model/PremiumFeaturesCache metabase.premium-features.models.premium-features-cache
:model/PythonLibrary metabase-enterprise.transforms-python.models.python-library
:model/Pulse metabase.pulse.models.pulse
:model/PulseCard metabase.pulse.models.pulse-card
diff --git a/src/metabase/premium_features/api.clj b/src/metabase/premium_features/api.clj
index 7c71faeb34c..bf40352a036 100644
--- a/src/metabase/premium_features/api.clj
+++ b/src/metabase/premium_features/api.clj
@@ -2,7 +2,9 @@
(:require
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
- [metabase.premium-features.core :as premium-features]))
+ [metabase.premium-features.core :as premium-features]
+ [metabase.premium-features.token-check :as token-check]
+ [ring.util.response :as response]))
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
;; use our API + we will need it when we make auto-TypeScript-signature generation happen
@@ -14,6 +16,17 @@
[]
(api/check-404 (premium-features/token-status)))
+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}
+(api.macros/defendpoint :post "/token/refresh"
+ "Clear all token caches and re-check the premium features token against the MetaStore.
+ Returns the fresh token status. Useful for the frontend after a purchase so that new features
+ are recognized without waiting for the cache TTL."
+ []
+ (token-check/clear-cache!)
+ (-> (api/check-404 (premium-features/token-status))
+ response/response
+ (assoc-in [:mb/cookies :cookie/premium-features-cache-timestamp] true)))
+
(def ^{:arglists '([request respond raise])} routes
"`/api/premium-features` routes."
(api.macros/ns-handler *ns* api/+check-superuser))
diff --git a/src/metabase/premium_features/init.clj b/src/metabase/premium_features/init.clj
index 3708aa217b7..ba462c880e2 100644
--- a/src/metabase/premium_features/init.clj
+++ b/src/metabase/premium_features/init.clj
@@ -2,4 +2,5 @@
(:require
[metabase.premium-features.core]
[metabase.premium-features.settings]
+ [metabase.premium-features.task.clear-token-cache]
[metabase.premium-features.task.send-metering]))
diff --git a/src/metabase/premium_features/models/premium_features_cache.clj b/src/metabase/premium_features/models/premium_features_cache.clj
new file mode 100644
index 00000000000..d9139f7bead
--- /dev/null
+++ b/src/metabase/premium_features/models/premium_features_cache.clj
@@ -0,0 +1,11 @@
+(ns metabase.premium-features.models.premium-features-cache
+ "Model for the `premium_features_token_cache` table, which stores hashes of premium features token check
+ results for cross-instance cache coordination. The actual token status is only held in memory."
+ (:require
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/PremiumFeaturesCache [_model] :premium_features_token_cache)
+
+(doto :model/PremiumFeaturesCache
+ (derive :metabase/model))
diff --git a/src/metabase/premium_features/task/clear_token_cache.clj b/src/metabase/premium_features/task/clear_token_cache.clj
new file mode 100644
index 00000000000..a1fd658e9b9
--- /dev/null
+++ b/src/metabase/premium_features/task/clear_token_cache.clj
@@ -0,0 +1,9 @@
+(ns metabase.premium-features.task.clear-token-cache
+ "Clears the premium features token cache on startup so the first token check always hits MetaStore fresh.
+ Without this, the cache table (which survives restarts) could serve stale feature data."
+ (:require
+ [metabase.premium-features.token-check :as token-check]
+ [metabase.startup.core :as startup]))
+
+(defmethod startup/def-startup-logic! ::clear-token-cache [_]
+ (token-check/clear-cache!))
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index aeb437c3e02..1547c3bd821 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -6,6 +6,8 @@
TODO -- We should move the settings in this namespace into [[metabase.premium-features.settings]]."
(:require
+ [buddy.core.codecs :as codecs]
+ [buddy.core.hash :as buddy-hash]
[clj-http.client :as http]
[clojure.core.memoize :as memoize]
[clojure.string :as str]
@@ -30,10 +32,7 @@
[metabase.util.string :as u.str]
[potemkin.types :as p]
[toucan2.connection :as t2.conn]
- [toucan2.core :as t2])
- (:import
- (com.google.common.cache CacheBuilder RemovalCause RemovalNotification)
- (java.util.concurrent TimeUnit)))
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -295,47 +294,6 @@
(def ^:dynamic *token-check-happening* "Var to prevent recursive calls to `fetch-token-status`" false)
-(p/defprotocol+ GracePeriod
- "A protocol for providing a grace period for token features in the event they are not fetchable for a little while."
- (save! [_ token features] "Save the features for a particular token.")
- (retrieve [_ token] "Attempt to retrieve features associated with a token. This is best effort, perhaps never set,
- perhaps has timed out. Possible this is nil."))
-
-(defn guava-cache-grace-period
- "Create a grace period of n units. This is just an expiring map using a guava cache. Note this is not sensitive to read times but to write times.
-
-
- (let [grace (guava-cache-grace-period 20 TimeUnit/MILLISECONDS)]
- (save! grace \"token\" #{\"features\"})
- (println \"found tokens?: \"
- (if (= (retrieve grace \"token\") #{\"features\"})
- \"🟒\"
- \"πŸ”΄\"))
- (Thread/sleep 40)
- (println \"expecting not to find tokens?: \"
- (if (retrieve grace \"token\")
- \"πŸ”΄\"
- \"🟒\")))
- found tokens?: 🟒
- expecting not to find tokens?: 🟒
- nil"
- [^long n ^TimeUnit units]
- (let [guava-cache (.. (CacheBuilder/newBuilder)
- (expireAfterWrite n units)
- (removalListener (fn [^RemovalNotification rn]
- (let [cause (.getCause rn)]
- (when (= RemovalCause/EXPIRED cause)
- (log/warnf "Removing token: %s from grace period cache"
- (u.str/mask (.getKey rn)))))))
- (build))]
- (reify GracePeriod
- (save! [_ token features] (.put guava-cache token features))
- (retrieve [_ token]
- (when token
- (let [value (.get guava-cache token (constantly ::not-present))]
- (when-not (identical? value ::not-present)
- value)))))))
-
(p/defprotocol+ TokenChecker
"Protocol for checking tokens with cache management."
(-check-token [this token]
@@ -350,9 +308,7 @@
(reify TokenChecker
(-check-token [_ token]
(decode-token* token))
- (-clear-cache! [_]
- ;; No cache to clear at this level
- nil)))
+ (-clear-cache! [_] nil)))
(defn circuit-breaker-token-checker
"Wraps a token checker with circuit breaker and timeout logic."
@@ -383,45 +339,136 @@
(catch dev.failsafe.FailsafeException e
(throw (.getCause e)))))))
(-clear-cache! [_]
- ;; No cache at this level, but delegate to wrapped checker
(-clear-cache! token-checker)))))
-(defn cached-token-checker
- "Wraps a token checker with TTL-based memoization."
- [token-checker {:keys [ttl-ms]}]
+(defn- hash-token
+ "SHA-256 hex hash of a token string. Used as cache table key so the raw token never appears in plaintext."
+ ^String [^String token]
+ (codecs/bytes->hex (buddy-hash/sha256 (.getBytes token "UTF-8"))))
+
+(defn- canonicalize-for-hash
+ "Prepare a token status map for deterministic hashing. Sorts map keys (via sorted-map)
+ and sorts sequential values that may arrive in arbitrary order from the MetaStore."
+ [m]
+ (cond
+ (map? m) (into (sorted-map) (update-vals m canonicalize-for-hash))
+ (sequential? m) (->> (map canonicalize-for-hash m)
+ (sort-by json/encode))
+ :else m))
+
+(defn- hash-token-status
+ "SHA-256 hex hash of a canonicalized, JSON-encoded token status map."
+ ^String [result]
+ (-> result canonicalize-for-hash json/encode (.getBytes "UTF-8") buddy-hash/sha256 codecs/bytes->hex))
+
+(defn- read-cache-from-db
+ "Read a cached token status hash from the premium_features_token_cache table. Returns nil if not found."
+ [token-hash]
+ (t2/select-one [:model/PremiumFeaturesCache :token_status_hash :updated_at] :token_hash token-hash))
+
+(defn- write-cache-to-db!
+ "Upsert a token status hash into the premium_features_token_cache table.
+ Uses insert-first to avoid TOCTOU race between instances."
+ [token-hash result-hash]
+ (let [now (t/offset-date-time)]
+ (try
+ (t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash
+ :token_status_hash result-hash
+ :updated_at now})
+ (catch Exception _
+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash
+ {:token_status_hash result-hash :updated_at now})))))
+
+(defn- clear-db-cache!
+ "Delete all rows from the premium_features_token_cache table."
+ []
+ (t2/delete! :model/PremiumFeaturesCache))
+
+(def ^:dynamic *testing-only-call-after-refresh*
+ "When non-nil, a zero-arg function called after async background refresh completes.
+ For testing only β€” do not use in production."
+ nil)
+
+(defn db-hash-aware-token-checker
+ "Wraps a token checker with a DB-hash-aware local cache.
+ The DB stores only a hash of the token status (never the actual features). The full token status
+ lives exclusively in the local in-memory cache. The DB hash is used to:
+ 1. Validate that the local cache is consistent with what other instances see
+ 2. Detect when another instance has refreshed and got different features (hash mismatch)
+
+ - `soft-ttl`: local entries younger than this are fresh; older triggers async refresh
+ - `hard-ttl`: local entries older than this are expired; synchronous fetch required
+ - `local-cache`: optional atom for the local cache (default: fresh atom). Exposed for testing.
+ - Hash mismatch or cold start always triggers a synchronous fetch."
+ [token-checker {:keys [soft-ttl hard-ttl local-cache]}]
+ (let [local-cache (or local-cache (atom {}))
+ refresh-in-progress? (atom false)]
+ (letfn [(do-refresh! [token token-hash]
+ (let [result (-check-token token-checker token)
+ result-hash (hash-token-status result)
+ now (t/instant)]
+ (write-cache-to-db! token-hash result-hash)
+ (swap! local-cache assoc token-hash {:result result
+ :result-hash result-hash
+ :updated-at now})
+ result))]
+ (reify TokenChecker
+ (-check-token [_ token]
+ (let [token-hash (hash-token token)
+ local-entry (get @local-cache token-hash)
+ db-row (read-cache-from-db token-hash)
+ now (t/instant)]
+ (if (and local-entry
+ db-row
+ (= (:result-hash local-entry) (:token_status_hash db-row)))
+ ;; Local cache matches DB hash β€” check TTLs
+ (let [age-start (:updated-at local-entry)]
+ (cond
+ ;; Fresh (< soft-ttl): return local value
+ (t/before? now (t/plus age-start soft-ttl))
+ (:result local-entry)
+
+ ;; Stale (soft..hard): return local + async refresh
+ (t/before? now (t/plus age-start hard-ttl))
+ (do
+ (when (compare-and-set! refresh-in-progress? false true)
+ (future
+ (try
+ (do-refresh! token token-hash)
+ (catch Exception e
+ (log/error e "Background premium features refresh failed"))
+ (finally
+ (reset! refresh-in-progress? false)
+ (when-let [after *testing-only-call-after-refresh*]
+ (after))))))
+ (:result local-entry))
+
+ ;; Expired (> hard-ttl): synchronous refresh
+ :else
+ (do-refresh! token token-hash)))
+
+ ;; No local cache, no DB row, or hash mismatch: synchronous fetch
+ (do-refresh! token token-hash))))
+ (-clear-cache! [_]
+ (reset! local-cache {})
+ (try
+ (clear-db-cache!)
+ (catch Exception e
+ (log/warn e "Failed to clear premium features cache table")))
+ (-clear-cache! token-checker))))))
+
+(defn local-cached-token-checker
+ "Wraps a token checker with short-lived in-memory TTL memoization.
+ Avoids hitting the DB on every request."
+ [token-checker {:keys [local-ttl]}]
(let [cached-check (memoize/ttl
(fn [token] (-check-token token-checker token))
- :ttl/threshold ttl-ms)]
+ :ttl/threshold (.toMillis ^java.time.Duration local-ttl))]
(reify TokenChecker
(-check-token [_ token]
(cached-check token))
(-clear-cache! [_]
- ;; Clear THIS layer's cache
(memoize/memo-clear! cached-check)
- ;; AND delegate to wrapped checker
- (-clear-cache! token-checker)))))
-
-(defn grace-period-token-checker
- "Wraps a token checker with grace period fallback logic."
- [token-checker {:keys [grace-period]}]
- (let [periodic-logger (memoize/ttl
- (fn [_token] (log/info "Using token from grace period"))
- :ttl/threshold (u/hours->ms 4))]
- (reify TokenChecker
- (-check-token [_ token]
- (try (let [response (-check-token token-checker token)]
- (save! grace-period token response)
- response)
- (catch Exception e
- (or (when-let [grace (retrieve grace-period token)]
- (periodic-logger token)
- grace)
- (throw e)))))
- (-clear-cache! [_]
- ;; The grace period itself doesn't need clearing (it expires naturally)
- ;; but we should clear the periodic logger cache
- (memoize/memo-clear! periodic-logger)
- ;; AND delegate to wrapped checker
(-clear-cache! token-checker)))))
(defn- error-catching-token-checker
@@ -445,40 +492,43 @@
(defn make-checker
"Make a token checker. Takes a base [[TokenChecker]] token checker and then arguments for the middleware-style
- wrapping [[TokenChecker]] arguments. "
- [{:keys [base circuit-breaker timeout-ms ttl-ms grace-period]
+ wrapping [[TokenChecker]] arguments.
+
+ `db-hash-local-cache` is optional and only provided during tests."
+ [{:keys [base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl db-hash-local-cache]
:or {base store-and-airgap-token-checker}}]
(when-not *customize-checker*
- (assert (and base circuit-breaker timeout-ms ttl-ms grace-period)
+ (assert (and base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl)
"Must provide all arguments for token checker"))
(cond-> base
- ;; don't do it too much
circuit-breaker
(circuit-breaker-token-checker {:circuit-breaker circuit-breaker
:timeout-ms timeout-ms})
- ;; hold onto results for a while
- ttl-ms
- (cached-token-checker {:ttl-ms ttl-ms})
- ;; in case of errors, if we have a recent response use it
- grace-period
- (grace-period-token-checker {:grace-period grace-period})
+
+ (and soft-ttl hard-ttl)
+ (db-hash-aware-token-checker {:soft-ttl soft-ttl
+ :hard-ttl hard-ttl
+ :local-cache db-hash-local-cache})
+
+ local-ttl
+ (local-cached-token-checker {:local-ttl local-ttl})
+
:always
(error-catching-token-checker)))
(def token-checker
- "The token checker. Combines http/airgapping validation, circuit breaking, grace periods, caching, and error
- handling."
+ "The token checker. Combines http/airgapping validation, circuit breaking, DB-hash-aware caching, and error handling."
(make-checker {:base store-and-airgap-token-checker
:circuit-breaker {:failure-threshold-ratio-in-period [10 10 (u/seconds->ms 60)]
-
:delay-ms (u/seconds->ms 30)
:success-threshold 1
:on-open (fn [_] (log/info "Engaging circuit breaker in token check"))
:on-half-open (fn [_] (log/info "In token check circuit breaker but attempting check"))
:on-close (fn [_] (log/info "Token Check restored"))}
- :timeout-ms (u/seconds->ms 10)
- :ttl-ms (u/hours->ms 12)
- :grace-period (guava-cache-grace-period 36 TimeUnit/HOURS)}))
+ :timeout-ms (u/seconds->ms 10)
+ :local-ttl (t/seconds 5)
+ :soft-ttl (t/hours 12)
+ :hard-ttl (t/hours 36)}))
(defn clear-cache!
"Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."
diff --git a/src/metabase/server/handler.clj b/src/metabase/server/handler.clj
index a81722b59bf..798d6a57e15 100644
--- a/src/metabase/server/handler.clj
+++ b/src/metabase/server/handler.clj
@@ -12,6 +12,7 @@
[metabase.server.middleware.metadata-provider-cache :as mw.mp-cache]
[metabase.server.middleware.misc :as mw.misc]
[metabase.server.middleware.offset-paging :as mw.offset-paging]
+ [metabase.server.middleware.premium-features-cache :as mw.pf-cache]
[metabase.server.middleware.request-id :as mw.request-id]
[metabase.server.middleware.security :as mw.security]
[metabase.server.middleware.session :as mw.session]
@@ -83,6 +84,7 @@
#'mw.session/reset-session-timeout ; Resets the timeout cookie for user activity to [[metabase.request.cookies/session-timeout]]
#'mw.session/bind-current-user ; Binds *current-user* and *current-user-id* if :metabase-user-id is non-nil
#'mw.session/wrap-current-user-info ; looks for :metabase-session-key and sets :metabase-user-id and other info if Session ID is valid
+ #'mw.pf-cache/wrap-premium-features-cache-check ; check cookie to refresh premium features cache if needed
#'mw.settings-cache/wrap-settings-cache-check ; check cookie to refresh settings cache if needed
#'analytics/embedding-mw ; reads sdk client headers, binds them to *client* and *version*, and tracks sdk-response metrics
#'mw.session/wrap-session-key ; looks for a Metabase Session ID and assoc as :metabase-session-key
diff --git a/src/metabase/server/middleware/premium_features_cache.clj b/src/metabase/server/middleware/premium_features_cache.clj
new file mode 100644
index 00000000000..a399748f087
--- /dev/null
+++ b/src/metabase/server/middleware/premium_features_cache.clj
@@ -0,0 +1,57 @@
+(ns metabase.server.middleware.premium-features-cache
+ "Ring middleware to propagate premium-features cache invalidation across instances via a cookie.
+ When one instance refreshes features (via POST /api/premium-features/token/refresh), it sets a cookie
+ with the current timestamp. Other instances see this cookie and clear all token caches so the
+ next feature check re-fetches from the MetaStore."
+ (:require
+ [metabase.premium-features.core :as premium-features]
+ [metabase.util.log :as log]
+ [ring.util.response :as response]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private cookie-name "metabase.PREMIUM_FEATURES_LAST_UPDATED")
+
+(def ^:private last-known-invalidation (atom nil))
+
+(defn- check-and-invalidate-features-cache
+ "Check if the cookie timestamp is newer than our last known invalidation, and if so, clear the cache."
+ [request]
+ (when-let [cookie-timestamp (get-in request [:cookies cookie-name :value])]
+ (let [last-known @last-known-invalidation]
+ (when (and cookie-timestamp
+ (or (nil? last-known)
+ (try (pos? (compare cookie-timestamp last-known))
+ (catch Exception _e
+ (log/infof "Strange premium features cookie value: %s" cookie-timestamp)
+ false))))
+ (log/info "Premium features cookie indicates cache is out of date. Clearing...")
+ (premium-features/clear-cache!)
+ (reset! last-known-invalidation cookie-timestamp)
+ ::invalidated))))
+
+(defn- maybe-set-premium-features-cookie
+ "Set the premium features last-updated cookie if this request triggered a refresh or propagated an invalidation."
+ [invalidated? response]
+ (if (or invalidated?
+ (-> response :mb/cookies :cookie/premium-features-cache-timestamp))
+ (let [timestamp (str (java.time.Instant/now))]
+ (reset! last-known-invalidation timestamp)
+ (response/set-cookie (update response :mb/cookies dissoc :cookie/premium-features-cache-timestamp)
+ cookie-name
+ timestamp
+ {:path "/"
+ :max-age (* 5 60)
+ :same-site :lax}))
+ response))
+
+(defn wrap-premium-features-cache-check
+ "Middleware that checks if the premium features cache needs to be invalidated based on a cookie.
+
+ When premium features are refreshed on one instance, the API sets a cookie with the current timestamp.
+ This middleware checks that cookie on subsequent requests and clears the local cache if needed,
+ ensuring consistency in multi-instance deployments."
+ [handler]
+ (fn [request respond raise]
+ (let [invalidated? (check-and-invalidate-features-cache request)]
+ (handler request (comp respond #(maybe-set-premium-features-cookie invalidated? %)) raise))))
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index 7b333a5ef6b..ef096a1adea 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -36,6 +36,7 @@
:model/DatabaseRouter
:model/Dependency
:model/PythonLibrary
+ :model/PremiumFeaturesCache
:model/Query
:model/QueryCache
:model/QueryExecution
diff --git a/test/metabase/premium_features/api_test.clj b/test/metabase/premium_features/api_test.clj
index 93d923ae113..c1d5c6ba9ca 100644
--- a/test/metabase/premium_features/api_test.clj
+++ b/test/metabase/premium_features/api_test.clj
@@ -1,6 +1,8 @@
(ns metabase.premium-features.api-test
(:require
+ [clj-http.cookies :as cookies]
[clojure.test :refer :all]
+ [metabase.premium-features.token-check :as token-check]
[metabase.test :as mt]))
(set! *warn-on-reflection* true)
@@ -23,3 +25,33 @@
(testing "returns 404 if no token is set"
(is (= "Not found."
(mt/user-http-request :crowberto :get 404 "premium-features/token/status")))))))
+
+(deftest post-token-refresh-test
+ (testing "POST /api/premium-features/token/refresh"
+ (mt/with-random-premium-token! [_token]
+ (testing "clears cache and returns fresh token status"
+ (let [cleared? (atom false)]
+ (with-redefs [token-check/clear-cache! (fn [] (reset! cleared? true))]
+ (is (=? {:valid true
+ :status "fake"
+ :features ["test" "fixture"]}
+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))
+ (is (true? @cleared?)))))
+
+ (testing "requires superusers"
+ (is (= "You don't have permissions to do that."
+ (mt/user-http-request :rasta :post 403 "premium-features/token/refresh")))))
+
+ (mt/with-temporary-setting-values [:premium-embedding-token nil]
+ (testing "returns 404 if no token is set"
+ (is (= "Not found."
+ (mt/user-http-request :crowberto :post 404 "premium-features/token/refresh")))))))
+
+(deftest token-refresh-sets-premium-features-cookie-test
+ (testing "POST /api/premium-features/token/refresh sets the premium-features-last-updated cookie"
+ (mt/with-random-premium-token! [_token]
+ (let [cs (cookies/cookie-store)]
+ (mt/user-real-request :crowberto :post 200 "premium-features/token/refresh"
+ {:request-options {:cookie-store cs}})
+ (let [pf-cookie (get (cookies/get-cookies cs) "metabase.PREMIUM_FEATURES_LAST_UPDATED")]
+ (is (some? pf-cookie) "No premium-features-last-updated cookie set"))))))
diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.clj
index fd54d7a3b0a..4f2615d3ade 100644
--- a/test/metabase/premium_features/token_check_test.clj
+++ b/test/metabase/premium_features/token_check_test.clj
@@ -3,19 +3,21 @@
[clj-http.client :as http]
[clj-http.core :as http.core]
[clojure.test :refer :all]
+ [java-time.api :as t]
[mb.hawk.parallel]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.core :as mdb]
[metabase.premium-features.core :as premium-features]
+ [metabase.premium-features.task.clear-token-cache]
[metabase.premium-features.test-util :as tu]
[metabase.premium-features.token-check :as token-check]
+ [metabase.startup.core :as startup]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
+ [metabase.util :as u]
[metabase.util.json :as json]
[metabase.util.malli.registry :as mr]
- [toucan2.core :as t2])
- (:import
- (java.util.concurrent TimeUnit)))
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -48,9 +50,10 @@
response (atom :error)
;; i've removed the circuit breaker from the stack. that sometimes shuts down the tests in a way we don't want
;; to exercise here
- checker (binding [token-check/*customize-checker* true]
- (token-check/make-checker {:ttl-ms 500
- :grace-period (token-check/guava-cache-grace-period 36 TimeUnit/HOURS)}))]
+ checker (binding [token-check/*customize-checker* true]
+ (token-check/make-checker {:local-ttl (t/seconds 5)
+ :soft-ttl (t/minutes 1)
+ :hard-ttl (t/minutes 2)}))]
(with-redefs [token-check/http-fetch (fn [& _]
(swap! call-count inc)
(case @response
@@ -71,17 +74,15 @@
(token-check/check-token (tu/random-token))))))
(deftest fetch-token-does-not-call-db-when-cached
- (testing "No DB calls are made when checking token status if the status is cached"
+ (testing "No DB calls are made when checking token status if the status is in local cache"
(let [token (tu/random-token)
_ (token-check/check-token token)
- ;; Sigh. This is really quite horrific. But we need some wiggle room here: any endpoint that gets some setting
- ;; inside it is going to check to see whether it's time for an update check. If it is, it'll hit the DB to see
- ;; when settings were last updated, and the count will be incremented. Therefore, let's do this a few times...
+ ;; The local cache has a 5s TTL, so repeated checks within that window should not hit the DB.
call-counts (repeatedly 3 (fn []
(t2/with-call-count [call-count]
(token-check/check-token token)
(call-count))))]
- ;; ... and then make sure that *some* of the times, we didn't hit the DB again.
+ ;; At least some of these should be zero (served from local in-memory cache)
(is (some zero? call-counts)))))
(deftest token-checker-test
@@ -89,9 +90,8 @@
behavior (atom :success)
call-count (atom 0)
good-response {:valid true
- :status :fake
- :features [:feature1 :feature2]}
- no-features {:valid false :status "Unable to validate token" :error-details "network issues!"}
+ :status "fake"
+ :features ["feature1" "feature2"]}
token-response (fn [_token]
(swap! call-count inc)
(case @behavior
@@ -107,8 +107,9 @@
:delay-ms 50
:success-threshold 1}
:timeout-ms 100
- :ttl-ms 200
- :grace-period (token-check/guava-cache-grace-period 1000 TimeUnit/MILLISECONDS)})]
+ :local-ttl (t/millis 50)
+ :soft-ttl (t/millis 200)
+ :hard-ttl (t/seconds 1)})]
(testing "when no information is present, hits the underlying"
(is (= good-response (token-check/-check-token checker token)))
(is (= 1 @call-count)))
@@ -119,12 +120,7 @@
(reset! behavior :error)
(Thread/sleep 200)
(dotimes [_ 1000] (token-check/-check-token checker token))
- (is (< @call-count 50))
- (testing "But we always get a good response from the grace period"
- (is (= good-response (token-check/-check-token checker token)))))
- (testing "But eventually grace period elapses"
- (Thread/sleep 1000)
- (is (= no-features (token-check/-check-token checker token))))
+ (is (< @call-count 50)))
(testing "When connectivity is restored we get successful token checks"
(reset! behavior :success)
(Thread/sleep 100) ;; wait for circuit breaker to open back up
@@ -132,8 +128,8 @@
(testing "App db not setup yet does not invoke the circuit breaker (#65294)"
(let [token (tu/random-token)
good-response {:valid true
- :status :fake
- :features [:feature1 :feature2]}
+ :status "fake"
+ :features ["feature1" "feature2"]}
token-response (fn [_token]
good-response)
checker (token-check/make-checker
@@ -145,8 +141,9 @@
:delay-ms 5000
:success-threshold 1}
:timeout-ms 100
- :ttl-ms 200
- :grace-period (token-check/guava-cache-grace-period 1000 TimeUnit/MILLISECONDS)})]
+ :local-ttl (t/millis 50)
+ :soft-ttl (t/millis 200)
+ :hard-ttl (t/seconds 1)})]
(with-redefs [mdb/db-is-set-up? (constantly false)]
(is (= {:valid false
:status "Unable to validate token"
@@ -161,73 +158,63 @@
(with-redefs [mdb/db-is-set-up? (constantly true)]
(is (= good-response (token-check/-check-token checker token))))))))
-(deftest grace-period-test
- (testing "implementation check"
- (let [token (tu/random-token)
- behavior (atom :success)
- success-token {:valid true :status :fake :features [:feature1 :feature2]}
- restored-token {:valid true :status :fake :features [:new-feature]}
- invalid {:valid false, :status "Unable to validate token", :error-details nil}
- underlying (reify token-check/TokenChecker
- (-check-token [_ t]
- (when (= t token)
- (case @behavior
- :success success-token
- :error (throw (ex-info "network troubles!" {:ka :boom}))
- :network-restored restored-token)))
- (-clear-cache! [_]))
- grace (binding [token-check/*customize-checker* true]
- (token-check/make-checker
- {:base underlying
- :grace-period (token-check/guava-cache-grace-period 20 TimeUnit/MILLISECONDS)}))]
- (is (= success-token (token-check/check-token grace token)))
- (is (= invalid (token-check/check-token grace (tu/random-token))))
- (testing "During errors"
- (reset! behavior :error)
- (is (= success-token (token-check/check-token grace token)))
- (is (= invalid (token-check/check-token grace (tu/random-token))))
- (Thread/sleep 40) ;; our simple one here has a grace period of 20 milliseconds
- (testing "but it expires"
- (is (= {:valid false :status "Unable to validate token" :error-details "network troubles!"}
- (token-check/check-token grace token)))))
- (testing "When good"
- (reset! behavior :success)
- (is (= success-token (token-check/check-token grace token))))
- (testing "We can enter the grace period"
- (reset! behavior :error)
- (is (= success-token (token-check/check-token grace token))))
- (testing "And then we immediately get new token when network restores"
- (reset! behavior :network-restored)
- (is (= restored-token (token-check/check-token grace token)))))))
-
-(deftest e2e
- (testing "token check hits the network"
- (let [token (tu/random-token)
- ;; todo: need to check that we use the grace period
- time-passes! token-check/clear-cache!
- call-count (atom 0)]
- (with-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- {:status 200
- :body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"})]
- (= {:valid true, :status "fake", :features ["fake" "features"]}
- (token-check/check-token token))
- (is (= 1 @call-count)))
- (with-redefs [token-check/http-fetch (fn [& _]
- (swap! call-count inc)
- (throw (ex-info "network failure!" {})))]
- (testing "Doesn't hit the network inside of cache duration"
- (is (= {:valid true, :status "fake", :features ["fake" "features"]}
- (token-check/check-token token)))
+(defn- age-cache-entry!
+ "Rewrite the `updated_at` timestamp for `token` in the cache table to be `age-ms` milliseconds in the past.
+ Also ages the local cache entry if a `local-cache` atom is provided."
+ ([token age-ms]
+ (age-cache-entry! token age-ms nil))
+ ([token age-ms local-cache]
+ (let [token-hash (#'token-check/hash-token token)
+ new-ts (t/minus (t/offset-date-time) (t/millis age-ms))]
+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash {:updated_at new-ts})
+ (when local-cache
+ (swap! local-cache update token-hash assoc :updated-at (t/minus (t/instant) (t/millis age-ms)))))))
+
+(deftest e2e-test
+ (testing "full stack: HTTP fetch β†’ DB-hash-aware cache β†’ error handling"
+ (let [token (tu/random-token)
+ call-count (atom 0)
+ good-body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"
+ good-resp {:valid true, :status "fake", :features ["fake" "features"]}
+ local-cache (atom {})
+ ;; no circuit breaker β€” it carries state between runs and causes flakes
+ checker (binding [token-check/*customize-checker* true]
+ (token-check/make-checker {:local-ttl (t/millis 50)
+ :soft-ttl (t/hours 12)
+ :hard-ttl (t/hours 36)
+ :db-hash-local-cache local-cache}))]
+ (try
+ (with-redefs [token-check/http-fetch (fn [& _]
+ (swap! call-count inc)
+ {:status 200 :body good-body})]
+ (is (= good-resp (token-check/check-token checker token)))
(is (= 1 @call-count)))
- (time-passes!) ;; we can clear the cache but don't have a great way to expire the grace period
- (testing "Hits the network periodically (12 hours)"
- (let [response (token-check/check-token token)]
- ;; called but we got network failure from redefs
- (is (= 2 @call-count))
- (testing "Grace period supplements errors"
- (is (= {:valid true, :status "fake", :features ["fake" "features"]}
- response)))))))))
+ (with-redefs [token-check/http-fetch (fn [& _]
+ (swap! call-count inc)
+ (throw (ex-info "network failure!" {})))]
+ (testing "Doesn't hit the network inside of cache duration"
+ ;; Wait for local-cached-token-checker TTL to expire, but DB-hash-aware cache is still fresh
+ (Thread/sleep 60)
+ (is (= good-resp (token-check/check-token checker token)))
+ (is (= 1 @call-count)))
+ (testing "Stale cache (past soft TTL, within hard TTL) still serves cached result"
+ (age-cache-entry! token (+ (u/hours->ms 12) 1000) local-cache)
+ (Thread/sleep 60) ;; expire local-cached-token-checker
+ (let [refresh-done (promise)]
+ (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]
+ (is (= good-resp (token-check/check-token checker token))))
+ ;; async refresh was attempted (and failed), but stale value was returned
+ (is (true? (deref refresh-done 5000 :timeout))))
+ (is (= 2 @call-count)))
+ (testing "Expired cache (past hard TTL) surfaces the error"
+ (age-cache-entry! token (+ (u/hours->ms 36) 1000) local-cache)
+ (Thread/sleep 60) ;; expire local-cached-token-checker
+ (is (= {:valid false
+ :status "Unable to validate token"
+ :error-details "network failure!"}
+ (token-check/check-token checker token)))))
+ (finally
+ (token-check/-clear-cache! checker))))))
(deftest token-status-setting-test
(testing "If a `premium-embedding-token` has been set, the `token-status` setting should return the response
@@ -372,3 +359,253 @@
(is (contains? stats :domains))
(is (contains? stats :embedding-dashboard-count))
(is (contains? stats :embedding-question-count)))))
+
+;;; ------------------------------------------------ db-hash-aware-token-checker ------------------------------------------------
+
+(defn- make-db-hash-aware-checker
+ "Helper: wraps a base token-response fn with DB-hash-aware caching. Returns the checker."
+ [token-response-fn {:keys [local-ttl soft-ttl hard-ttl]
+ :or {local-ttl (t/millis 50)}}]
+ (binding [token-check/*customize-checker* true]
+ (token-check/make-checker
+ {:base (reify token-check/TokenChecker
+ (-check-token [_ token] (token-response-fn token))
+ (-clear-cache! [_]))
+ :local-ttl local-ttl
+ :soft-ttl soft-ttl
+ :hard-ttl hard-ttl})))
+
+(deftest db-hash-aware-token-checker-fresh-hit-test
+ (testing "Fresh cache entry (< soft-ttl) is returned without hitting the underlying checker"
+ (let [call-count (atom 0)
+ good-response {:valid true :status "OK" :features ["sandboxes"]}
+ checker (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count inc)
+ good-response)
+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
+ token (tu/random-token)]
+ (try
+ ;; First call: cache miss β†’ delegate
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (= 1 @call-count))
+ ;; Second call: fresh cache hit β†’ no delegate
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (= 1 @call-count))
+ (finally
+ (token-check/-clear-cache! checker))))))
+
+(deftest db-hash-aware-token-checker-stale-async-refresh-test
+ (testing "Stale entry (soft..hard) returns cached value and triggers async refresh"
+ (let [call-count (atom 0)
+ good-response {:valid true :status "OK" :features ["sandboxes"]}
+ updated-response {:valid true :status "OK" :features ["sandboxes" "audit-app"]}
+ response-atom (atom good-response)
+ refresh-done (promise)
+ checker (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count inc)
+ @response-atom)
+ ;; soft=1ms so it's immediately stale, hard=10s so not expired
+ {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/seconds 10)})
+ token (tu/random-token)]
+ (try
+ ;; First call: cache miss β†’ delegate
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (= 1 @call-count))
+ ;; Let the entry become stale (both local and DB)
+ (Thread/sleep 10)
+ ;; Switch to updated response for the async refresh
+ (reset! response-atom updated-response)
+ ;; Second call: stale β†’ returns old cached value, kicks off async refresh
+ (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (true? @refresh-done)))
+ ;; Wait for the async refresh to complete
+ (is (not= :timeout (deref refresh-done 5000 :timeout)))
+ ;; Expire local cache
+ (Thread/sleep 10)
+ ;; Third call: should now see the refreshed value
+ (is (= updated-response (token-check/-check-token checker token)))
+ (finally
+ (token-check/-clear-cache! checker))))))
+
+(deftest db-hash-aware-token-checker-expired-sync-test
+ (testing "Expired entry (> hard-ttl) delegates synchronously"
+ (let [call-count (atom 0)
+ good-response {:valid true :status "OK" :features ["sandboxes"]}
+ checker (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count inc)
+ good-response)
+ ;; Both TTLs = 1ms so entry expires immediately
+ {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})
+ token (tu/random-token)]
+ (try
+ ;; First call: miss
+ (token-check/-check-token checker token)
+ (is (= 1 @call-count))
+ (Thread/sleep 5)
+ ;; Second call: expired β†’ synchronous delegate
+ (token-check/-check-token checker token)
+ (is (= 2 @call-count))
+ (finally
+ (token-check/-clear-cache! checker))))))
+
+(deftest db-hash-aware-token-checker-multi-token-test
+ (testing "Different tokens are cached independently"
+ (let [call-count (atom 0)
+ checker (make-db-hash-aware-checker
+ (fn [token]
+ (swap! call-count inc)
+ {:valid true :status "OK" :features [(str "feat-" token)]})
+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
+ token-a (tu/random-token)
+ token-b (tu/random-token)]
+ (try
+ (let [result-a (token-check/-check-token checker token-a)]
+ (is (= [(str "feat-" token-a)] (:features result-a))))
+ (let [result-b (token-check/-check-token checker token-b)]
+ (is (= [(str "feat-" token-b)] (:features result-b))))
+ (is (= 2 @call-count))
+ ;; Both cached now
+ (token-check/-check-token checker token-a)
+ (token-check/-check-token checker token-b)
+ (is (= 2 @call-count))
+ (finally
+ (token-check/-clear-cache! checker))))))
+
+(deftest db-hash-aware-token-checker-clear-cache-test
+ (testing "clear-cache! removes both local and DB cache"
+ (let [checker (make-db-hash-aware-checker
+ (fn [_token]
+ {:valid true :status "OK" :features ["sandboxes"]})
+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
+ token (tu/random-token)]
+ (token-check/-check-token checker token)
+ (let [token-hash (#'token-check/hash-token token)]
+ (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))
+ (token-check/-clear-cache! checker)
+ (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))))))
+
+(deftest db-hash-aware-token-checker-invalid-cached-test
+ (testing "Invalid token responses are also cached in table to avoid hammering MetaStore"
+ (let [call-count (atom 0)
+ checker (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count inc)
+ {:valid false :status "Token does not exist."})
+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
+ token (tu/random-token)]
+ (try
+ (is (= {:valid false :status "Token does not exist."}
+ (token-check/-check-token checker token)))
+ (is (= 1 @call-count))
+ ;; Second call should use cache
+ (is (= {:valid false :status "Token does not exist."}
+ (token-check/-check-token checker token)))
+ (is (= 1 @call-count))
+ (finally
+ (token-check/-clear-cache! checker))))))
+
+(deftest db-hash-aware-token-checker-cross-instance-sync-test
+ (testing "When instance A refreshes and gets new features, instance B detects the hash mismatch and re-fetches"
+ (let [call-count-a (atom 0)
+ call-count-b (atom 0)
+ old-response {:valid true :status "OK" :features ["sandboxes"]}
+ new-response {:valid true :status "OK" :features ["sandboxes" "audit-app"]}
+ response-a (atom old-response)
+ response-b (atom old-response)
+ checker-a (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count-a inc)
+ @response-a)
+ {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})
+ checker-b (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count-b inc)
+ @response-b)
+ {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})
+ token (tu/random-token)]
+ (try
+ ;; Both instances populate their caches with old-response
+ (is (= old-response (token-check/-check-token checker-a token)))
+ (is (= old-response (token-check/-check-token checker-b token)))
+ (is (= 1 @call-count-a))
+ (is (= 1 @call-count-b))
+ ;; Instance A gets new features and refreshes (expired TTL forces sync refresh)
+ (reset! response-a new-response)
+ (Thread/sleep 5) ;; expire checker-a's local + DB TTLs
+ (is (= new-response (token-check/-check-token checker-a token)))
+ (is (= 2 @call-count-a))
+ ;; Instance B has a long TTL, but the DB hash now differs from its local hash
+ ;; So it should detect the mismatch and do a synchronous re-fetch
+ (reset! response-b new-response)
+ (Thread/sleep 5) ;; expire local-cached-token-checker's 1ms TTL
+ (is (= new-response (token-check/-check-token checker-b token)))
+ (is (= 2 @call-count-b))
+ (finally
+ (token-check/-clear-cache! checker-a)
+ (token-check/-clear-cache! checker-b))))))
+
+(deftest db-hash-aware-token-checker-exception-propagation-test
+ (testing "When delegate throws on expired/missing, exception propagates through error-catching"
+ (let [checker (make-db-hash-aware-checker
+ (fn [_token]
+ (throw (ex-info "MetaStore unreachable" {})))
+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})
+ token (tu/random-token)]
+ ;; error-catching wraps it, so we get the error-details response
+ (is (= {:valid false
+ :status "Unable to validate token"
+ :error-details "MetaStore unreachable"}
+ (token-check/-check-token checker token))))))
+
+(deftest db-hash-aware-token-checker-tamper-resistance-test
+ (testing "Tampering with the DB hash triggers a re-fetch from MetaStore, not a cache hit"
+ (let [call-count (atom 0)
+ good-response {:valid true :status "OK" :features ["sandboxes"]}
+ checker (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count inc)
+ good-response)
+ {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})
+ token (tu/random-token)
+ token-hash (#'token-check/hash-token token)]
+ (try
+ ;; Populate the cache
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (= 1 @call-count))
+ ;; Tamper with the DB hash β€” simulate someone modifying the row
+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash
+ {:token_status_hash "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"})
+ ;; Expire local-cached-token-checker
+ (Thread/sleep 5)
+ ;; Next check should detect hash mismatch and re-fetch from MetaStore
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (= 2 @call-count))
+ (finally
+ (token-check/-clear-cache! checker))))))
+
+(deftest startup-clears-token-cache-test
+ (testing "The startup hook clears the token cache table so the first check after boot hits MetaStore"
+ (let [call-count (atom 0)
+ good-response {:valid true :status "OK" :features ["sandboxes"]}
+ checker (make-db-hash-aware-checker
+ (fn [_token]
+ (swap! call-count inc)
+ good-response)
+ {:soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})
+ token (tu/random-token)]
+ (try
+ ;; Populate the cache
+ (token-check/-check-token checker token)
+ (is (= 1 @call-count))
+ (let [token-hash (#'token-check/hash-token token)]
+ (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))
+ ;; Run the startup hook (clears the global token-checker's cache, including the table)
+ (startup/def-startup-logic! :metabase.premium-features.task.clear-token-cache/clear-token-cache)
+ ;; Table should be cleared
+ (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash))))
+ (finally
+ (token-check/-clear-cache! checker))))))