The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Implement DB-level caching of premium features (#69241)
John Swanson34837bc9bbbe
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 77a5fa49e65..a1bad371174 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1025,6 +1025,7 @@eventsinternal-statssettings+ startuptaskutil}}diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex 1089be59da4..c88c7496266 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -97,6 +97,7 @@"PermissionsGroupMembership""PermissionsRevision""PersistedInfo"+ "PremiumFeaturesCache""Pulse""PulseCard""PulseChannel"diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 899db5d9d41..5dd17327901 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -72,6 +72,7 @@:model/PermissionsGroupMembership:model/PermissionsRevision:model/PersistedInfo+ :model/PremiumFeaturesCache:model/Pulse:model/PulseCard:model/PulseChanneldiff --git a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsx b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsxindex bd142eb3145..7fc4466e505 100644--- a/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsx+++ b/enterprise/frontend/src/metabase-enterprise/metabot/components/MetabotAdmin/MetabotPurchasePage/MetabotPurchasePage.unit.spec.tsx@@ -7,6 +7,7 @@ import {setupStoreEEBillingEndpoint,setupStoreEECloudAddOnsEndpoint,setupStoreEETieredMetabotAI,+ setupTokenRefreshEndpoint,} from "__support__/server-mocks";import { renderWithProviders, screen } from "__support__/ui";import {@@ -103,6 +104,7 @@ const setup = async ({simulate_http_get_error,);setupStoreEETieredMetabotAI(simulate_http_post_error);+ setupTokenRefreshEndpoint();renderWithProviders(<MetabotPurchasePage />, {// Calling `setupPropertiesEndpoints` above is not enough; we also need to set `storeInitialState`:diff --git a/frontend/src/metabase/api/index.ts b/frontend/src/metabase/api/index.tsindex af76ea8a9de..d2e6177e4b4 100644--- a/frontend/src/metabase/api/index.ts+++ b/frontend/src/metabase/api/index.ts@@ -29,6 +29,7 @@ export * from "./notification";export * from "./parameters";export * from "./permission";export * from "./persist";+export * from "./premium-features";export * from "./revision";export * from "./search";export * from "./segment";diff --git a/frontend/src/metabase/api/premium-features.ts b/frontend/src/metabase/api/premium-features.tsnew file mode 100644index 00000000000..6a7cbaa1872--- /dev/null+++ b/frontend/src/metabase/api/premium-features.ts@@ -0,0 +1,19 @@+import type { TokenStatus } from "metabase-types/api";++import { Api } from "./api";+import { invalidateTags, tag } from "./tags";++export const premiumFeaturesApi = Api.injectEndpoints({+ endpoints: (builder) => ({+ refreshTokenStatus: builder.mutation<TokenStatus, void>({+ query: () => ({+ method: "POST",+ url: "/api/premium-features/token/refresh",+ }),+ invalidatesTags: (_, error) =>+ invalidateTags(error, [tag("session-properties")]),+ }),+ }),+});++export const { useRefreshTokenStatusMutation } = premiumFeaturesApi;diff --git a/frontend/src/metabase/api/utils/use-token-refresh.ts b/frontend/src/metabase/api/utils/use-token-refresh.tsindex cff7742dd5c..01e30ffd043 100644--- a/frontend/src/metabase/api/utils/use-token-refresh.ts+++ b/frontend/src/metabase/api/utils/use-token-refresh.ts@@ -1,6 +1,10 @@-import { useEffect } from "react";+import { useCallback, useEffect } from "react";-import { Api, useGetSettingsQuery } from "metabase/api";+import {+ Api,+ useGetSettingsQuery,+ useRefreshTokenStatusMutation,+} from "metabase/api";import { useDispatch } from "metabase/lib/redux";import type { TokenStatusFeature } from "metabase-types/api";@@ -47,6 +51,18 @@ export function useTokenRefreshUntil(/* in order to force this hook to re-run on every request, even if the response data is the same, we can't destructure only the data prop from this hook, as is the pattern in many components */const res = useGetSettingsQuery();const dispatch = useDispatch();+ const [refreshTokenStatus] = useRefreshTokenStatusMutation();++ const refreshToken = useCallback(async () => {+ // Bust the server-side cache first; the mutation's invalidatesTags will+ // also invalidate session-properties on success, but we do it in finally+ // too so the UI updates even if the request fails.+ try {+ await refreshTokenStatus().unwrap();+ } finally {+ dispatch(Api.util.invalidateTags(["session-properties"]));+ }+ }, [dispatch, refreshTokenStatus]);useEffect(() => {if (skip) {@@ -57,10 +73,8 @@ export function useTokenRefreshUntil(const isTokenFeatureMissing = !tokenStatusFeatures?.includes(tokenFeature);if (isTokenFeatureMissing) {- const timeout = setTimeout(() => {- dispatch(Api.util.invalidateTags(["session-properties"]));- }, intervalMs);+ const timeout = setTimeout(refreshToken, intervalMs);return () => clearTimeout(timeout);}- }, [res, dispatch, tokenFeature, intervalMs, skip]);+ }, [res, dispatch, tokenFeature, intervalMs, skip, refreshToken]);}diff --git a/frontend/test/__support__/server-mocks/premium-features.ts b/frontend/test/__support__/server-mocks/premium-features.tsindex 7e6760e50fc..9343e172d5a 100644--- a/frontend/test/__support__/server-mocks/premium-features.ts+++ b/frontend/test/__support__/server-mocks/premium-features.ts@@ -26,6 +26,12 @@ export const setupTokenStatusEndpointEmpty = () => {});};+export const setupTokenRefreshEndpoint = () => {+ fetchMock.post("path:/api/premium-features/token/refresh", 200, {+ name: "premium-token-refresh",+ });+};+export const setupTokenActivationEndpoint = ({success,status,diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 3cbc52ef5ae..9e30e59a2ad 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -2986,6 +2986,35 @@ databaseChangeLog:constraints:nullable: true+ - changeSet:+ id: v59.2026-02-07T17:00:00+ author: johnswanson+ comment: Create premium_features_token_cache table for cross-instance token cache+ changes:+ - createTable:+ tableName: premium_features_token_cache+ remarks: Caches premium features token check results, shared across instances+ columns:+ - column:+ name: token_hash+ type: varchar(64)+ remarks: SHA-256 hex hash of the premium token+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: token_status_hash+ type: varchar(64)+ remarks: SHA-256 hex hash of the JSON-encoded token status+ constraints:+ nullable: false+ - column:+ name: updated_at+ type: ${timestamp_type}+ remarks: When this cache entry was last refreshed+ constraints:+ nullable: false+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex d5e92c1d903..6597aaa8d63 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -73,6 +73,7 @@:model/PermissionsGroupMembership metabase.permissions.models.permissions-group-membership:model/PermissionsRevision metabase.permissions.models.permissions-revision:model/PersistedInfo metabase.model-persistence.models.persisted-info+ :model/PremiumFeaturesCache metabase.premium-features.models.premium-features-cache:model/PythonLibrary metabase-enterprise.transforms-python.models.python-library:model/Pulse metabase.pulse.models.pulse:model/PulseCard metabase.pulse.models.pulse-carddiff --git a/src/metabase/premium_features/api.clj b/src/metabase/premium_features/api.cljindex 7c71faeb34c..bf40352a036 100644--- a/src/metabase/premium_features/api.clj+++ b/src/metabase/premium_features/api.clj@@ -2,7 +2,9 @@(:require[metabase.api.common :as api][metabase.api.macros :as api.macros]- [metabase.premium-features.core :as premium-features]))+ [metabase.premium-features.core :as premium-features]+ [metabase.premium-features.token-check :as token-check]+ [ring.util.response :as response]));; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to;; use our API + we will need it when we make auto-TypeScript-signature generation happen@@ -14,6 +16,17 @@[](api/check-404 (premium-features/token-status)))+#_{:clj-kondo/ignore [:metabase/validate-defendpoint-has-response-schema]}+(api.macros/defendpoint :post "/token/refresh"+ "Clear all token caches and re-check the premium features token against the MetaStore.+ Returns the fresh token status. Useful for the frontend after a purchase so that new features+ are recognized without waiting for the cache TTL."+ []+ (token-check/clear-cache!)+ (-> (api/check-404 (premium-features/token-status))+ response/response+ (assoc-in [:mb/cookies :cookie/premium-features-cache-timestamp] true)))+(def ^{:arglists '([request respond raise])} routes"`/api/premium-features` routes."(api.macros/ns-handler *ns* api/+check-superuser))diff --git a/src/metabase/premium_features/init.clj b/src/metabase/premium_features/init.cljindex 3708aa217b7..ba462c880e2 100644--- a/src/metabase/premium_features/init.clj+++ b/src/metabase/premium_features/init.clj@@ -2,4 +2,5 @@(:require[metabase.premium-features.core][metabase.premium-features.settings]+ [metabase.premium-features.task.clear-token-cache][metabase.premium-features.task.send-metering]))diff --git a/src/metabase/premium_features/models/premium_features_cache.clj b/src/metabase/premium_features/models/premium_features_cache.cljnew file mode 100644index 00000000000..d9139f7bead--- /dev/null+++ b/src/metabase/premium_features/models/premium_features_cache.clj@@ -0,0 +1,11 @@+(ns metabase.premium-features.models.premium-features-cache+ "Model for the `premium_features_token_cache` table, which stores hashes of premium features token check+ results for cross-instance cache coordination. The actual token status is only held in memory."+ (:require+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/PremiumFeaturesCache [_model] :premium_features_token_cache)++(doto :model/PremiumFeaturesCache+ (derive :metabase/model))diff --git a/src/metabase/premium_features/task/clear_token_cache.clj b/src/metabase/premium_features/task/clear_token_cache.cljnew file mode 100644index 00000000000..a1fd658e9b9--- /dev/null+++ b/src/metabase/premium_features/task/clear_token_cache.clj@@ -0,0 +1,9 @@+(ns metabase.premium-features.task.clear-token-cache+ "Clears the premium features token cache on startup so the first token check always hits MetaStore fresh.+ Without this, the cache table (which survives restarts) could serve stale feature data."+ (:require+ [metabase.premium-features.token-check :as token-check]+ [metabase.startup.core :as startup]))++(defmethod startup/def-startup-logic! ::clear-token-cache [_]+ (token-check/clear-cache!))diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex aeb437c3e02..1547c3bd821 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -6,6 +6,8 @@TODO -- We should move the settings in this namespace into [[metabase.premium-features.settings]]."(:require+ [buddy.core.codecs :as codecs]+ [buddy.core.hash :as buddy-hash][clj-http.client :as http][clojure.core.memoize :as memoize][clojure.string :as str]@@ -30,10 +32,7 @@[metabase.util.string :as u.str][potemkin.types :as p][toucan2.connection :as t2.conn]- [toucan2.core :as t2])- (:import- (com.google.common.cache CacheBuilder RemovalCause RemovalNotification)- (java.util.concurrent TimeUnit)))+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -295,47 +294,6 @@(def ^:dynamic *token-check-happening* "Var to prevent recursive calls to `fetch-token-status`" false)-(p/defprotocol+ GracePeriod- "A protocol for providing a grace period for token features in the event they are not fetchable for a little while."- (save! [_ token features] "Save the features for a particular token.")- (retrieve [_ token] "Attempt to retrieve features associated with a token. This is best effort, perhaps never set,- perhaps has timed out. Possible this is nil."))--(defn guava-cache-grace-period- "Create a grace period of n units. This is just an expiring map using a guava cache. Note this is not sensitive to read times but to write times.--- (let [grace (guava-cache-grace-period 20 TimeUnit/MILLISECONDS)]- (save! grace \"token\" #{\"features\"})- (println \"found tokens?: \"- (if (= (retrieve grace \"token\") #{\"features\"})- \"π’\"- \"π΄\"))- (Thread/sleep 40)- (println \"expecting not to find tokens?: \"- (if (retrieve grace \"token\")- \"π΄\"- \"π’\")))- found tokens?: π’- expecting not to find tokens?: π’- nil"- [^long n ^TimeUnit units]- (let [guava-cache (.. (CacheBuilder/newBuilder)- (expireAfterWrite n units)- (removalListener (fn [^RemovalNotification rn]- (let [cause (.getCause rn)]- (when (= RemovalCause/EXPIRED cause)- (log/warnf "Removing token: %s from grace period cache"- (u.str/mask (.getKey rn)))))))- (build))]- (reify GracePeriod- (save! [_ token features] (.put guava-cache token features))- (retrieve [_ token]- (when token- (let [value (.get guava-cache token (constantly ::not-present))]- (when-not (identical? value ::not-present)- value)))))))-(p/defprotocol+ TokenChecker"Protocol for checking tokens with cache management."(-check-token [this token]@@ -350,9 +308,7 @@(reify TokenChecker(-check-token [_ token](decode-token* token))- (-clear-cache! [_]- ;; No cache to clear at this level- nil)))+ (-clear-cache! [_] nil)))(defn circuit-breaker-token-checker"Wraps a token checker with circuit breaker and timeout logic."@@ -383,45 +339,136 @@(catch dev.failsafe.FailsafeException e(throw (.getCause e)))))))(-clear-cache! [_]- ;; No cache at this level, but delegate to wrapped checker(-clear-cache! token-checker)))))-(defn cached-token-checker- "Wraps a token checker with TTL-based memoization."- [token-checker {:keys [ttl-ms]}]+(defn- hash-token+ "SHA-256 hex hash of a token string. Used as cache table key so the raw token never appears in plaintext."+ ^String [^String token]+ (codecs/bytes->hex (buddy-hash/sha256 (.getBytes token "UTF-8"))))++(defn- canonicalize-for-hash+ "Prepare a token status map for deterministic hashing. Sorts map keys (via sorted-map)+ and sorts sequential values that may arrive in arbitrary order from the MetaStore."+ [m]+ (cond+ (map? m) (into (sorted-map) (update-vals m canonicalize-for-hash))+ (sequential? m) (->> (map canonicalize-for-hash m)+ (sort-by json/encode))+ :else m))++(defn- hash-token-status+ "SHA-256 hex hash of a canonicalized, JSON-encoded token status map."+ ^String [result]+ (-> result canonicalize-for-hash json/encode (.getBytes "UTF-8") buddy-hash/sha256 codecs/bytes->hex))++(defn- read-cache-from-db+ "Read a cached token status hash from the premium_features_token_cache table. Returns nil if not found."+ [token-hash]+ (t2/select-one [:model/PremiumFeaturesCache :token_status_hash :updated_at] :token_hash token-hash))++(defn- write-cache-to-db!+ "Upsert a token status hash into the premium_features_token_cache table.+ Uses insert-first to avoid TOCTOU race between instances."+ [token-hash result-hash]+ (let [now (t/offset-date-time)]+ (try+ (t2/insert! :model/PremiumFeaturesCache {:token_hash token-hash+ :token_status_hash result-hash+ :updated_at now})+ (catch Exception _+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash+ {:token_status_hash result-hash :updated_at now})))))++(defn- clear-db-cache!+ "Delete all rows from the premium_features_token_cache table."+ []+ (t2/delete! :model/PremiumFeaturesCache))++(def ^:dynamic *testing-only-call-after-refresh*+ "When non-nil, a zero-arg function called after async background refresh completes.+ For testing only β do not use in production."+ nil)++(defn db-hash-aware-token-checker+ "Wraps a token checker with a DB-hash-aware local cache.+ The DB stores only a hash of the token status (never the actual features). The full token status+ lives exclusively in the local in-memory cache. The DB hash is used to:+ 1. Validate that the local cache is consistent with what other instances see+ 2. Detect when another instance has refreshed and got different features (hash mismatch)++ - `soft-ttl`: local entries younger than this are fresh; older triggers async refresh+ - `hard-ttl`: local entries older than this are expired; synchronous fetch required+ - `local-cache`: optional atom for the local cache (default: fresh atom). Exposed for testing.+ - Hash mismatch or cold start always triggers a synchronous fetch."+ [token-checker {:keys [soft-ttl hard-ttl local-cache]}]+ (let [local-cache (or local-cache (atom {}))+ refresh-in-progress? (atom false)]+ (letfn [(do-refresh! [token token-hash]+ (let [result (-check-token token-checker token)+ result-hash (hash-token-status result)+ now (t/instant)]+ (write-cache-to-db! token-hash result-hash)+ (swap! local-cache assoc token-hash {:result result+ :result-hash result-hash+ :updated-at now})+ result))]+ (reify TokenChecker+ (-check-token [_ token]+ (let [token-hash (hash-token token)+ local-entry (get @local-cache token-hash)+ db-row (read-cache-from-db token-hash)+ now (t/instant)]+ (if (and local-entry+ db-row+ (= (:result-hash local-entry) (:token_status_hash db-row)))+ ;; Local cache matches DB hash β check TTLs+ (let [age-start (:updated-at local-entry)]+ (cond+ ;; Fresh (< soft-ttl): return local value+ (t/before? now (t/plus age-start soft-ttl))+ (:result local-entry)++ ;; Stale (soft..hard): return local + async refresh+ (t/before? now (t/plus age-start hard-ttl))+ (do+ (when (compare-and-set! refresh-in-progress? false true)+ (future+ (try+ (do-refresh! token token-hash)+ (catch Exception e+ (log/error e "Background premium features refresh failed"))+ (finally+ (reset! refresh-in-progress? false)+ (when-let [after *testing-only-call-after-refresh*]+ (after))))))+ (:result local-entry))++ ;; Expired (> hard-ttl): synchronous refresh+ :else+ (do-refresh! token token-hash)))++ ;; No local cache, no DB row, or hash mismatch: synchronous fetch+ (do-refresh! token token-hash))))+ (-clear-cache! [_]+ (reset! local-cache {})+ (try+ (clear-db-cache!)+ (catch Exception e+ (log/warn e "Failed to clear premium features cache table")))+ (-clear-cache! token-checker))))))++(defn local-cached-token-checker+ "Wraps a token checker with short-lived in-memory TTL memoization.+ Avoids hitting the DB on every request."+ [token-checker {:keys [local-ttl]}](let [cached-check (memoize/ttl(fn [token] (-check-token token-checker token))- :ttl/threshold ttl-ms)]+ :ttl/threshold (.toMillis ^java.time.Duration local-ttl))](reify TokenChecker(-check-token [_ token](cached-check token))(-clear-cache! [_]- ;; Clear THIS layer's cache(memoize/memo-clear! cached-check)- ;; AND delegate to wrapped checker- (-clear-cache! token-checker)))))--(defn grace-period-token-checker- "Wraps a token checker with grace period fallback logic."- [token-checker {:keys [grace-period]}]- (let [periodic-logger (memoize/ttl- (fn [_token] (log/info "Using token from grace period"))- :ttl/threshold (u/hours->ms 4))]- (reify TokenChecker- (-check-token [_ token]- (try (let [response (-check-token token-checker token)]- (save! grace-period token response)- response)- (catch Exception e- (or (when-let [grace (retrieve grace-period token)]- (periodic-logger token)- grace)- (throw e)))))- (-clear-cache! [_]- ;; The grace period itself doesn't need clearing (it expires naturally)- ;; but we should clear the periodic logger cache- (memoize/memo-clear! periodic-logger)- ;; AND delegate to wrapped checker(-clear-cache! token-checker)))))(defn- error-catching-token-checker@@ -445,40 +492,43 @@(defn make-checker"Make a token checker. Takes a base [[TokenChecker]] token checker and then arguments for the middleware-style- wrapping [[TokenChecker]] arguments. "- [{:keys [base circuit-breaker timeout-ms ttl-ms grace-period]+ wrapping [[TokenChecker]] arguments.++ `db-hash-local-cache` is optional and only provided during tests."+ [{:keys [base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl db-hash-local-cache]:or {base store-and-airgap-token-checker}}](when-not *customize-checker*- (assert (and base circuit-breaker timeout-ms ttl-ms grace-period)+ (assert (and base circuit-breaker timeout-ms local-ttl soft-ttl hard-ttl)"Must provide all arguments for token checker"))(cond-> base- ;; don't do it too muchcircuit-breaker(circuit-breaker-token-checker {:circuit-breaker circuit-breaker:timeout-ms timeout-ms})- ;; hold onto results for a while- ttl-ms- (cached-token-checker {:ttl-ms ttl-ms})- ;; in case of errors, if we have a recent response use it- grace-period- (grace-period-token-checker {:grace-period grace-period})++ (and soft-ttl hard-ttl)+ (db-hash-aware-token-checker {:soft-ttl soft-ttl+ :hard-ttl hard-ttl+ :local-cache db-hash-local-cache})++ local-ttl+ (local-cached-token-checker {:local-ttl local-ttl})+:always(error-catching-token-checker)))(def token-checker- "The token checker. Combines http/airgapping validation, circuit breaking, grace periods, caching, and error- handling."+ "The token checker. Combines http/airgapping validation, circuit breaking, DB-hash-aware caching, and error handling."(make-checker {:base store-and-airgap-token-checker:circuit-breaker {:failure-threshold-ratio-in-period [10 10 (u/seconds->ms 60)]-:delay-ms (u/seconds->ms 30):success-threshold 1:on-open (fn [_] (log/info "Engaging circuit breaker in token check")):on-half-open (fn [_] (log/info "In token check circuit breaker but attempting check")):on-close (fn [_] (log/info "Token Check restored"))}- :timeout-ms (u/seconds->ms 10)- :ttl-ms (u/hours->ms 12)- :grace-period (guava-cache-grace-period 36 TimeUnit/HOURS)}))+ :timeout-ms (u/seconds->ms 10)+ :local-ttl (t/seconds 5)+ :soft-ttl (t/hours 12)+ :hard-ttl (t/hours 36)}))(defn clear-cache!"Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."diff --git a/src/metabase/server/handler.clj b/src/metabase/server/handler.cljindex a81722b59bf..798d6a57e15 100644--- a/src/metabase/server/handler.clj+++ b/src/metabase/server/handler.clj@@ -12,6 +12,7 @@[metabase.server.middleware.metadata-provider-cache :as mw.mp-cache][metabase.server.middleware.misc :as mw.misc][metabase.server.middleware.offset-paging :as mw.offset-paging]+ [metabase.server.middleware.premium-features-cache :as mw.pf-cache][metabase.server.middleware.request-id :as mw.request-id][metabase.server.middleware.security :as mw.security][metabase.server.middleware.session :as mw.session]@@ -83,6 +84,7 @@#'mw.session/reset-session-timeout ; Resets the timeout cookie for user activity to [[metabase.request.cookies/session-timeout]]#'mw.session/bind-current-user ; Binds *current-user* and *current-user-id* if :metabase-user-id is non-nil#'mw.session/wrap-current-user-info ; looks for :metabase-session-key and sets :metabase-user-id and other info if Session ID is valid+ #'mw.pf-cache/wrap-premium-features-cache-check ; check cookie to refresh premium features cache if needed#'mw.settings-cache/wrap-settings-cache-check ; check cookie to refresh settings cache if needed#'analytics/embedding-mw ; reads sdk client headers, binds them to *client* and *version*, and tracks sdk-response metrics#'mw.session/wrap-session-key ; looks for a Metabase Session ID and assoc as :metabase-session-keydiff --git a/src/metabase/server/middleware/premium_features_cache.clj b/src/metabase/server/middleware/premium_features_cache.cljnew file mode 100644index 00000000000..a399748f087--- /dev/null+++ b/src/metabase/server/middleware/premium_features_cache.clj@@ -0,0 +1,57 @@+(ns metabase.server.middleware.premium-features-cache+ "Ring middleware to propagate premium-features cache invalidation across instances via a cookie.+ When one instance refreshes features (via POST /api/premium-features/token/refresh), it sets a cookie+ with the current timestamp. Other instances see this cookie and clear all token caches so the+ next feature check re-fetches from the MetaStore."+ (:require+ [metabase.premium-features.core :as premium-features]+ [metabase.util.log :as log]+ [ring.util.response :as response]))++(set! *warn-on-reflection* true)++(def ^:private cookie-name "metabase.PREMIUM_FEATURES_LAST_UPDATED")++(def ^:private last-known-invalidation (atom nil))++(defn- check-and-invalidate-features-cache+ "Check if the cookie timestamp is newer than our last known invalidation, and if so, clear the cache."+ [request]+ (when-let [cookie-timestamp (get-in request [:cookies cookie-name :value])]+ (let [last-known @last-known-invalidation]+ (when (and cookie-timestamp+ (or (nil? last-known)+ (try (pos? (compare cookie-timestamp last-known))+ (catch Exception _e+ (log/infof "Strange premium features cookie value: %s" cookie-timestamp)+ false))))+ (log/info "Premium features cookie indicates cache is out of date. Clearing...")+ (premium-features/clear-cache!)+ (reset! last-known-invalidation cookie-timestamp)+ ::invalidated))))++(defn- maybe-set-premium-features-cookie+ "Set the premium features last-updated cookie if this request triggered a refresh or propagated an invalidation."+ [invalidated? response]+ (if (or invalidated?+ (-> response :mb/cookies :cookie/premium-features-cache-timestamp))+ (let [timestamp (str (java.time.Instant/now))]+ (reset! last-known-invalidation timestamp)+ (response/set-cookie (update response :mb/cookies dissoc :cookie/premium-features-cache-timestamp)+ cookie-name+ timestamp+ {:path "/"+ :max-age (* 5 60)+ :same-site :lax}))+ response))++(defn wrap-premium-features-cache-check+ "Middleware that checks if the premium features cache needs to be invalidated based on a cookie.++ When premium features are refreshed on one instance, the API sets a cookie with the current timestamp.+ This middleware checks that cookie on subsequent requests and clears the local cache if needed,+ ensuring consistency in multi-instance deployments."+ [handler]+ (fn [request respond raise]+ (let [invalidated? (check-and-invalidate-features-cache request)]+ (handler request (comp respond #(maybe-set-premium-features-cookie invalidated? %)) raise))))diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex 7b333a5ef6b..ef096a1adea 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -36,6 +36,7 @@:model/DatabaseRouter:model/Dependency:model/PythonLibrary+ :model/PremiumFeaturesCache:model/Query:model/QueryCache:model/QueryExecutiondiff --git a/test/metabase/premium_features/api_test.clj b/test/metabase/premium_features/api_test.cljindex 93d923ae113..c1d5c6ba9ca 100644--- a/test/metabase/premium_features/api_test.clj+++ b/test/metabase/premium_features/api_test.clj@@ -1,6 +1,8 @@(ns metabase.premium-features.api-test(:require+ [clj-http.cookies :as cookies][clojure.test :refer :all]+ [metabase.premium-features.token-check :as token-check][metabase.test :as mt]))(set! *warn-on-reflection* true)@@ -23,3 +25,33 @@(testing "returns 404 if no token is set"(is (= "Not found."(mt/user-http-request :crowberto :get 404 "premium-features/token/status")))))))++(deftest post-token-refresh-test+ (testing "POST /api/premium-features/token/refresh"+ (mt/with-random-premium-token! [_token]+ (testing "clears cache and returns fresh token status"+ (let [cleared? (atom false)]+ (with-redefs [token-check/clear-cache! (fn [] (reset! cleared? true))]+ (is (=? {:valid true+ :status "fake"+ :features ["test" "fixture"]}+ (mt/user-http-request :crowberto :post 200 "premium-features/token/refresh")))+ (is (true? @cleared?)))))++ (testing "requires superusers"+ (is (= "You don't have permissions to do that."+ (mt/user-http-request :rasta :post 403 "premium-features/token/refresh")))))++ (mt/with-temporary-setting-values [:premium-embedding-token nil]+ (testing "returns 404 if no token is set"+ (is (= "Not found."+ (mt/user-http-request :crowberto :post 404 "premium-features/token/refresh")))))))++(deftest token-refresh-sets-premium-features-cookie-test+ (testing "POST /api/premium-features/token/refresh sets the premium-features-last-updated cookie"+ (mt/with-random-premium-token! [_token]+ (let [cs (cookies/cookie-store)]+ (mt/user-real-request :crowberto :post 200 "premium-features/token/refresh"+ {:request-options {:cookie-store cs}})+ (let [pf-cookie (get (cookies/get-cookies cs) "metabase.PREMIUM_FEATURES_LAST_UPDATED")]+ (is (some? pf-cookie) "No premium-features-last-updated cookie set"))))))diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.cljindex fd54d7a3b0a..4f2615d3ade 100644--- a/test/metabase/premium_features/token_check_test.clj+++ b/test/metabase/premium_features/token_check_test.clj@@ -3,19 +3,21 @@[clj-http.client :as http][clj-http.core :as http.core][clojure.test :refer :all]+ [java-time.api :as t][mb.hawk.parallel][metabase.app-db.connection :as mdb.connection][metabase.app-db.core :as mdb][metabase.premium-features.core :as premium-features]+ [metabase.premium-features.task.clear-token-cache][metabase.premium-features.test-util :as tu][metabase.premium-features.token-check :as token-check]+ [metabase.startup.core :as startup][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util :as u][metabase.util.json :as json][metabase.util.malli.registry :as mr]- [toucan2.core :as t2])- (:import- (java.util.concurrent TimeUnit)))+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -48,9 +50,10 @@response (atom :error);; i've removed the circuit breaker from the stack. that sometimes shuts down the tests in a way we don't want;; to exercise here- checker (binding [token-check/*customize-checker* true]- (token-check/make-checker {:ttl-ms 500- :grace-period (token-check/guava-cache-grace-period 36 TimeUnit/HOURS)}))]+ checker (binding [token-check/*customize-checker* true]+ (token-check/make-checker {:local-ttl (t/seconds 5)+ :soft-ttl (t/minutes 1)+ :hard-ttl (t/minutes 2)}))](with-redefs [token-check/http-fetch (fn [& _](swap! call-count inc)(case @response@@ -71,17 +74,15 @@(token-check/check-token (tu/random-token))))))(deftest fetch-token-does-not-call-db-when-cached- (testing "No DB calls are made when checking token status if the status is cached"+ (testing "No DB calls are made when checking token status if the status is in local cache"(let [token (tu/random-token)_ (token-check/check-token token)- ;; Sigh. This is really quite horrific. But we need some wiggle room here: any endpoint that gets some setting- ;; inside it is going to check to see whether it's time for an update check. If it is, it'll hit the DB to see- ;; when settings were last updated, and the count will be incremented. Therefore, let's do this a few times...+ ;; The local cache has a 5s TTL, so repeated checks within that window should not hit the DB.call-counts (repeatedly 3 (fn [](t2/with-call-count [call-count](token-check/check-token token)(call-count))))]- ;; ... and then make sure that *some* of the times, we didn't hit the DB again.+ ;; At least some of these should be zero (served from local in-memory cache)(is (some zero? call-counts)))))(deftest token-checker-test@@ -89,9 +90,8 @@behavior (atom :success)call-count (atom 0)good-response {:valid true- :status :fake- :features [:feature1 :feature2]}- no-features {:valid false :status "Unable to validate token" :error-details "network issues!"}+ :status "fake"+ :features ["feature1" "feature2"]}token-response (fn [_token](swap! call-count inc)(case @behavior@@ -107,8 +107,9 @@:delay-ms 50:success-threshold 1}:timeout-ms 100- :ttl-ms 200- :grace-period (token-check/guava-cache-grace-period 1000 TimeUnit/MILLISECONDS)})]+ :local-ttl (t/millis 50)+ :soft-ttl (t/millis 200)+ :hard-ttl (t/seconds 1)})](testing "when no information is present, hits the underlying"(is (= good-response (token-check/-check-token checker token)))(is (= 1 @call-count)))@@ -119,12 +120,7 @@(reset! behavior :error)(Thread/sleep 200)(dotimes [_ 1000] (token-check/-check-token checker token))- (is (< @call-count 50))- (testing "But we always get a good response from the grace period"- (is (= good-response (token-check/-check-token checker token)))))- (testing "But eventually grace period elapses"- (Thread/sleep 1000)- (is (= no-features (token-check/-check-token checker token))))+ (is (< @call-count 50)))(testing "When connectivity is restored we get successful token checks"(reset! behavior :success)(Thread/sleep 100) ;; wait for circuit breaker to open back up@@ -132,8 +128,8 @@(testing "App db not setup yet does not invoke the circuit breaker (#65294)"(let [token (tu/random-token)good-response {:valid true- :status :fake- :features [:feature1 :feature2]}+ :status "fake"+ :features ["feature1" "feature2"]}token-response (fn [_token]good-response)checker (token-check/make-checker@@ -145,8 +141,9 @@:delay-ms 5000:success-threshold 1}:timeout-ms 100- :ttl-ms 200- :grace-period (token-check/guava-cache-grace-period 1000 TimeUnit/MILLISECONDS)})]+ :local-ttl (t/millis 50)+ :soft-ttl (t/millis 200)+ :hard-ttl (t/seconds 1)})](with-redefs [mdb/db-is-set-up? (constantly false)](is (= {:valid false:status "Unable to validate token"@@ -161,73 +158,63 @@(with-redefs [mdb/db-is-set-up? (constantly true)](is (= good-response (token-check/-check-token checker token))))))))-(deftest grace-period-test- (testing "implementation check"- (let [token (tu/random-token)- behavior (atom :success)- success-token {:valid true :status :fake :features [:feature1 :feature2]}- restored-token {:valid true :status :fake :features [:new-feature]}- invalid {:valid false, :status "Unable to validate token", :error-details nil}- underlying (reify token-check/TokenChecker- (-check-token [_ t]- (when (= t token)- (case @behavior- :success success-token- :error (throw (ex-info "network troubles!" {:ka :boom}))- :network-restored restored-token)))- (-clear-cache! [_]))- grace (binding [token-check/*customize-checker* true]- (token-check/make-checker- {:base underlying- :grace-period (token-check/guava-cache-grace-period 20 TimeUnit/MILLISECONDS)}))]- (is (= success-token (token-check/check-token grace token)))- (is (= invalid (token-check/check-token grace (tu/random-token))))- (testing "During errors"- (reset! behavior :error)- (is (= success-token (token-check/check-token grace token)))- (is (= invalid (token-check/check-token grace (tu/random-token))))- (Thread/sleep 40) ;; our simple one here has a grace period of 20 milliseconds- (testing "but it expires"- (is (= {:valid false :status "Unable to validate token" :error-details "network troubles!"}- (token-check/check-token grace token)))))- (testing "When good"- (reset! behavior :success)- (is (= success-token (token-check/check-token grace token))))- (testing "We can enter the grace period"- (reset! behavior :error)- (is (= success-token (token-check/check-token grace token))))- (testing "And then we immediately get new token when network restores"- (reset! behavior :network-restored)- (is (= restored-token (token-check/check-token grace token)))))))--(deftest e2e- (testing "token check hits the network"- (let [token (tu/random-token)- ;; todo: need to check that we use the grace period- time-passes! token-check/clear-cache!- call-count (atom 0)]- (with-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- {:status 200- :body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"})]- (= {:valid true, :status "fake", :features ["fake" "features"]}- (token-check/check-token token))- (is (= 1 @call-count)))- (with-redefs [token-check/http-fetch (fn [& _]- (swap! call-count inc)- (throw (ex-info "network failure!" {})))]- (testing "Doesn't hit the network inside of cache duration"- (is (= {:valid true, :status "fake", :features ["fake" "features"]}- (token-check/check-token token)))+(defn- age-cache-entry!+ "Rewrite the `updated_at` timestamp for `token` in the cache table to be `age-ms` milliseconds in the past.+ Also ages the local cache entry if a `local-cache` atom is provided."+ ([token age-ms]+ (age-cache-entry! token age-ms nil))+ ([token age-ms local-cache]+ (let [token-hash (#'token-check/hash-token token)+ new-ts (t/minus (t/offset-date-time) (t/millis age-ms))]+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash {:updated_at new-ts})+ (when local-cache+ (swap! local-cache update token-hash assoc :updated-at (t/minus (t/instant) (t/millis age-ms)))))))++(deftest e2e-test+ (testing "full stack: HTTP fetch β DB-hash-aware cache β error handling"+ (let [token (tu/random-token)+ call-count (atom 0)+ good-body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"+ good-resp {:valid true, :status "fake", :features ["fake" "features"]}+ local-cache (atom {})+ ;; no circuit breaker β it carries state between runs and causes flakes+ checker (binding [token-check/*customize-checker* true]+ (token-check/make-checker {:local-ttl (t/millis 50)+ :soft-ttl (t/hours 12)+ :hard-ttl (t/hours 36)+ :db-hash-local-cache local-cache}))]+ (try+ (with-redefs [token-check/http-fetch (fn [& _]+ (swap! call-count inc)+ {:status 200 :body good-body})]+ (is (= good-resp (token-check/check-token checker token)))(is (= 1 @call-count)))- (time-passes!) ;; we can clear the cache but don't have a great way to expire the grace period- (testing "Hits the network periodically (12 hours)"- (let [response (token-check/check-token token)]- ;; called but we got network failure from redefs- (is (= 2 @call-count))- (testing "Grace period supplements errors"- (is (= {:valid true, :status "fake", :features ["fake" "features"]}- response)))))))))+ (with-redefs [token-check/http-fetch (fn [& _]+ (swap! call-count inc)+ (throw (ex-info "network failure!" {})))]+ (testing "Doesn't hit the network inside of cache duration"+ ;; Wait for local-cached-token-checker TTL to expire, but DB-hash-aware cache is still fresh+ (Thread/sleep 60)+ (is (= good-resp (token-check/check-token checker token)))+ (is (= 1 @call-count)))+ (testing "Stale cache (past soft TTL, within hard TTL) still serves cached result"+ (age-cache-entry! token (+ (u/hours->ms 12) 1000) local-cache)+ (Thread/sleep 60) ;; expire local-cached-token-checker+ (let [refresh-done (promise)]+ (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]+ (is (= good-resp (token-check/check-token checker token))))+ ;; async refresh was attempted (and failed), but stale value was returned+ (is (true? (deref refresh-done 5000 :timeout))))+ (is (= 2 @call-count)))+ (testing "Expired cache (past hard TTL) surfaces the error"+ (age-cache-entry! token (+ (u/hours->ms 36) 1000) local-cache)+ (Thread/sleep 60) ;; expire local-cached-token-checker+ (is (= {:valid false+ :status "Unable to validate token"+ :error-details "network failure!"}+ (token-check/check-token checker token)))))+ (finally+ (token-check/-clear-cache! checker))))))(deftest token-status-setting-test(testing "If a `premium-embedding-token` has been set, the `token-status` setting should return the response@@ -372,3 +359,253 @@(is (contains? stats :domains))(is (contains? stats :embedding-dashboard-count))(is (contains? stats :embedding-question-count)))))++;;; ------------------------------------------------ db-hash-aware-token-checker ------------------------------------------------++(defn- make-db-hash-aware-checker+ "Helper: wraps a base token-response fn with DB-hash-aware caching. Returns the checker."+ [token-response-fn {:keys [local-ttl soft-ttl hard-ttl]+ :or {local-ttl (t/millis 50)}}]+ (binding [token-check/*customize-checker* true]+ (token-check/make-checker+ {:base (reify token-check/TokenChecker+ (-check-token [_ token] (token-response-fn token))+ (-clear-cache! [_]))+ :local-ttl local-ttl+ :soft-ttl soft-ttl+ :hard-ttl hard-ttl})))++(deftest db-hash-aware-token-checker-fresh-hit-test+ (testing "Fresh cache entry (< soft-ttl) is returned without hitting the underlying checker"+ (let [call-count (atom 0)+ good-response {:valid true :status "OK" :features ["sandboxes"]}+ checker (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count inc)+ good-response)+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})+ token (tu/random-token)]+ (try+ ;; First call: cache miss β delegate+ (is (= good-response (token-check/-check-token checker token)))+ (is (= 1 @call-count))+ ;; Second call: fresh cache hit β no delegate+ (is (= good-response (token-check/-check-token checker token)))+ (is (= 1 @call-count))+ (finally+ (token-check/-clear-cache! checker))))))++(deftest db-hash-aware-token-checker-stale-async-refresh-test+ (testing "Stale entry (soft..hard) returns cached value and triggers async refresh"+ (let [call-count (atom 0)+ good-response {:valid true :status "OK" :features ["sandboxes"]}+ updated-response {:valid true :status "OK" :features ["sandboxes" "audit-app"]}+ response-atom (atom good-response)+ refresh-done (promise)+ checker (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count inc)+ @response-atom)+ ;; soft=1ms so it's immediately stale, hard=10s so not expired+ {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/seconds 10)})+ token (tu/random-token)]+ (try+ ;; First call: cache miss β delegate+ (is (= good-response (token-check/-check-token checker token)))+ (is (= 1 @call-count))+ ;; Let the entry become stale (both local and DB)+ (Thread/sleep 10)+ ;; Switch to updated response for the async refresh+ (reset! response-atom updated-response)+ ;; Second call: stale β returns old cached value, kicks off async refresh+ (binding [token-check/*testing-only-call-after-refresh* #(deliver refresh-done true)]+ (is (= good-response (token-check/-check-token checker token)))+ (is (true? @refresh-done)))+ ;; Wait for the async refresh to complete+ (is (not= :timeout (deref refresh-done 5000 :timeout)))+ ;; Expire local cache+ (Thread/sleep 10)+ ;; Third call: should now see the refreshed value+ (is (= updated-response (token-check/-check-token checker token)))+ (finally+ (token-check/-clear-cache! checker))))))++(deftest db-hash-aware-token-checker-expired-sync-test+ (testing "Expired entry (> hard-ttl) delegates synchronously"+ (let [call-count (atom 0)+ good-response {:valid true :status "OK" :features ["sandboxes"]}+ checker (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count inc)+ good-response)+ ;; Both TTLs = 1ms so entry expires immediately+ {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})+ token (tu/random-token)]+ (try+ ;; First call: miss+ (token-check/-check-token checker token)+ (is (= 1 @call-count))+ (Thread/sleep 5)+ ;; Second call: expired β synchronous delegate+ (token-check/-check-token checker token)+ (is (= 2 @call-count))+ (finally+ (token-check/-clear-cache! checker))))))++(deftest db-hash-aware-token-checker-multi-token-test+ (testing "Different tokens are cached independently"+ (let [call-count (atom 0)+ checker (make-db-hash-aware-checker+ (fn [token]+ (swap! call-count inc)+ {:valid true :status "OK" :features [(str "feat-" token)]})+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})+ token-a (tu/random-token)+ token-b (tu/random-token)]+ (try+ (let [result-a (token-check/-check-token checker token-a)]+ (is (= [(str "feat-" token-a)] (:features result-a))))+ (let [result-b (token-check/-check-token checker token-b)]+ (is (= [(str "feat-" token-b)] (:features result-b))))+ (is (= 2 @call-count))+ ;; Both cached now+ (token-check/-check-token checker token-a)+ (token-check/-check-token checker token-b)+ (is (= 2 @call-count))+ (finally+ (token-check/-clear-cache! checker))))))++(deftest db-hash-aware-token-checker-clear-cache-test+ (testing "clear-cache! removes both local and DB cache"+ (let [checker (make-db-hash-aware-checker+ (fn [_token]+ {:valid true :status "OK" :features ["sandboxes"]})+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})+ token (tu/random-token)]+ (token-check/-check-token checker token)+ (let [token-hash (#'token-check/hash-token token)]+ (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))+ (token-check/-clear-cache! checker)+ (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))))))++(deftest db-hash-aware-token-checker-invalid-cached-test+ (testing "Invalid token responses are also cached in table to avoid hammering MetaStore"+ (let [call-count (atom 0)+ checker (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count inc)+ {:valid false :status "Token does not exist."})+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})+ token (tu/random-token)]+ (try+ (is (= {:valid false :status "Token does not exist."}+ (token-check/-check-token checker token)))+ (is (= 1 @call-count))+ ;; Second call should use cache+ (is (= {:valid false :status "Token does not exist."}+ (token-check/-check-token checker token)))+ (is (= 1 @call-count))+ (finally+ (token-check/-clear-cache! checker))))))++(deftest db-hash-aware-token-checker-cross-instance-sync-test+ (testing "When instance A refreshes and gets new features, instance B detects the hash mismatch and re-fetches"+ (let [call-count-a (atom 0)+ call-count-b (atom 0)+ old-response {:valid true :status "OK" :features ["sandboxes"]}+ new-response {:valid true :status "OK" :features ["sandboxes" "audit-app"]}+ response-a (atom old-response)+ response-b (atom old-response)+ checker-a (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count-a inc)+ @response-a)+ {:local-ttl (t/millis 1) :soft-ttl (t/millis 1) :hard-ttl (t/millis 1)})+ checker-b (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count-b inc)+ @response-b)+ {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})+ token (tu/random-token)]+ (try+ ;; Both instances populate their caches with old-response+ (is (= old-response (token-check/-check-token checker-a token)))+ (is (= old-response (token-check/-check-token checker-b token)))+ (is (= 1 @call-count-a))+ (is (= 1 @call-count-b))+ ;; Instance A gets new features and refreshes (expired TTL forces sync refresh)+ (reset! response-a new-response)+ (Thread/sleep 5) ;; expire checker-a's local + DB TTLs+ (is (= new-response (token-check/-check-token checker-a token)))+ (is (= 2 @call-count-a))+ ;; Instance B has a long TTL, but the DB hash now differs from its local hash+ ;; So it should detect the mismatch and do a synchronous re-fetch+ (reset! response-b new-response)+ (Thread/sleep 5) ;; expire local-cached-token-checker's 1ms TTL+ (is (= new-response (token-check/-check-token checker-b token)))+ (is (= 2 @call-count-b))+ (finally+ (token-check/-clear-cache! checker-a)+ (token-check/-clear-cache! checker-b))))))++(deftest db-hash-aware-token-checker-exception-propagation-test+ (testing "When delegate throws on expired/missing, exception propagates through error-catching"+ (let [checker (make-db-hash-aware-checker+ (fn [_token]+ (throw (ex-info "MetaStore unreachable" {})))+ {:soft-ttl (t/minutes 1) :hard-ttl (t/minutes 2)})+ token (tu/random-token)]+ ;; error-catching wraps it, so we get the error-details response+ (is (= {:valid false+ :status "Unable to validate token"+ :error-details "MetaStore unreachable"}+ (token-check/-check-token checker token))))))++(deftest db-hash-aware-token-checker-tamper-resistance-test+ (testing "Tampering with the DB hash triggers a re-fetch from MetaStore, not a cache hit"+ (let [call-count (atom 0)+ good-response {:valid true :status "OK" :features ["sandboxes"]}+ checker (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count inc)+ good-response)+ {:local-ttl (t/millis 1) :soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})+ token (tu/random-token)+ token-hash (#'token-check/hash-token token)]+ (try+ ;; Populate the cache+ (is (= good-response (token-check/-check-token checker token)))+ (is (= 1 @call-count))+ ;; Tamper with the DB hash β simulate someone modifying the row+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash+ {:token_status_hash "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"})+ ;; Expire local-cached-token-checker+ (Thread/sleep 5)+ ;; Next check should detect hash mismatch and re-fetch from MetaStore+ (is (= good-response (token-check/-check-token checker token)))+ (is (= 2 @call-count))+ (finally+ (token-check/-clear-cache! checker))))))++(deftest startup-clears-token-cache-test+ (testing "The startup hook clears the token cache table so the first check after boot hits MetaStore"+ (let [call-count (atom 0)+ good-response {:valid true :status "OK" :features ["sandboxes"]}+ checker (make-db-hash-aware-checker+ (fn [_token]+ (swap! call-count inc)+ good-response)+ {:soft-ttl (t/minutes 10) :hard-ttl (t/minutes 20)})+ token (tu/random-token)]+ (try+ ;; Populate the cache+ (token-check/-check-token checker token)+ (is (= 1 @call-count))+ (let [token-hash (#'token-check/hash-token token)]+ (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))+ ;; Run the startup hook (clears the global token-checker's cache, including the table)+ (startup/def-startup-logic! :metabase.premium-features.task.clear-token-cache/clear-token-cache)+ ;; Table should be cleared+ (is (nil? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash))))+ (finally+ (token-check/-clear-cache! checker))))))