Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve metabase.PREMIUM_FEATURES_LAST_UPDATED cookie handling (#82514)

Nathan Voxland752667252c20
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index 862fdd6a76f..ccb93850eb0 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -37,6 +37,7 @@
quotas
TokenStatus
clear-cache!
+ clear-local-cache!
token-check-url
transform-metered-as
transform-stats]
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index 4f2e2c52d3b..59aa1a3962e 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -335,8 +335,13 @@
(-check-token [this token]
"Check a token and return TokenStatus map. May throw exceptions on failure.")
(-clear-cache! [this]
- "Clear any caches in this checker and any wrapped checkers.
- Returns nil. Implementations should delegate to wrapped checkers."))
+ "Clear any caches in this checker and any wrapped checkers, including the shared DB cache.
+ Returns nil. Implementations should delegate to wrapped checkers.")
+ (-clear-local-cache! [this]
+ "Clear only the in-process memoization in this checker and any wrapped checkers. The shared DB cache and the
+ DB-hash-validated local cache are left alone, so the next check re-validates against the DB and only refetches
+ from the MetaStore if another instance really did get a different token status. Safe to trigger from untrusted
+ input. Returns nil."))
(def store-and-airgap-token-checker
"Creates a basic token checker that handles HTTP requests and airgap tokens.
@@ -344,7 +349,8 @@
(reify TokenChecker
(-check-token [_ token]
(decode-token* token))
- (-clear-cache! [_] nil)))
+ (-clear-cache! [_] nil)
+ (-clear-local-cache! [_] nil)))
(defn circuit-breaker-token-checker
"Wraps a token checker with circuit breaker and timeout logic."
@@ -378,7 +384,9 @@
(catch dev.failsafe.FailsafeException e
(throw (or (.getCause e) e)))))))
(-clear-cache! [_]
- (-clear-cache! token-checker)))))
+ (-clear-cache! token-checker))
+ (-clear-local-cache! [_]
+ (-clear-local-cache! token-checker)))))
(defn- hash-token
"SHA-256 hex hash of a token string. Used as cache table key so the raw token never appears in plaintext."
@@ -520,7 +528,11 @@
(reset! local-cache {})
(when (app-db/db-is-set-up?)
(clear-db-cache!))
- (-clear-cache! token-checker))))))
+ (-clear-cache! token-checker))
+ (-clear-local-cache! [_]
+ ;; `local-cache` is validated against the DB hash on every check, so it self-corrects; dropping it would
+ ;; only force a synchronous MetaStore fetch.
+ (-clear-local-cache! token-checker))))))
(defn local-cached-token-checker
"Wraps a token checker with short-lived in-memory TTL memoization.
@@ -534,7 +546,10 @@
(cached-check token))
(-clear-cache! [_]
(memoize/memo-clear! cached-check)
- (-clear-cache! token-checker)))))
+ (-clear-cache! token-checker))
+ (-clear-local-cache! [_]
+ (memoize/memo-clear! cached-check)
+ (-clear-local-cache! token-checker)))))
(defn- error-catching-token-checker
[token-checker]
@@ -549,7 +564,8 @@
:canonical? false
:status (tru "Unable to validate token")
:error-details (.getMessage e)})))
- (-clear-cache! [_] (-clear-cache! token-checker))))
+ (-clear-cache! [_] (-clear-cache! token-checker))
+ (-clear-local-cache! [_] (-clear-local-cache! token-checker))))
#_{:clj-kondo/ignore [:metabase/discourage-dynamic-vars]}
(def ^:dynamic *customize-checker*
@@ -607,6 +623,12 @@
[]
(-clear-cache! token-checker))
+(defn clear-local-cache!
+ "Clear only this process's in-memory token memoization; see [[-clear-local-cache!]]. Unlike [[clear-cache!]] this
+ never touches the shared DB cache, so it is safe to trigger from an untrusted request."
+ []
+ (-clear-local-cache! token-checker))
+
(defn check-token
"Public entrypoint to the token checking."
([token] (check-token token-checker token))
diff --git a/src/metabase/server/middleware/premium_features_cache.clj b/src/metabase/server/middleware/premium_features_cache.clj
index 36c5fe77fe2..ea8c747a2d5 100644
--- a/src/metabase/server/middleware/premium_features_cache.clj
+++ b/src/metabase/server/middleware/premium_features_cache.clj
@@ -1,57 +1,93 @@
(ns metabase.server.middleware.premium-features-cache
"Ring middleware to propagate premium-features cache invalidation across instances via a cookie.
When one instance refreshes features (via POST /api/premium-features/token/refresh), it sets a cookie
- with the current timestamp. Other instances see this cookie and clear all token caches so the
- next feature check re-fetches from the MetaStore."
+ with the current timestamp. Other instances see this cookie and drop their in-process token caches so
+ the next feature check re-validates against the shared DB cache, and only re-fetches from the MetaStore
+ if that shows the other instance really did get a different token status.
+
+ The cookie is client-controlled and arrives on unauthenticated requests, so it is a hint, never an
+ authority: it must parse as a timestamp, must not be in the future, must be newer than the last
+ invalidation this instance saw, and can only ever clear this instance's in-memory caches. It never deletes
+ the deployment's shared DB cache."
(:require
[metabase.premium-features.core :as premium-features]
[metabase.util.log :as log]
- [ring.util.response :as response]))
+ [ring.util.response :as response])
+ (:import
+ (java.time Duration Instant)
+ (java.time.format DateTimeParseException)))
(set! *warn-on-reflection* true)
(def ^:private cookie-name "metabase.PREMIUM_FEATURES_LAST_UPDATED")
-(def ^:private last-known-invalidation (atom nil))
+(def ^:private ^Duration max-clock-skew
+ "How far in the future a cookie timestamp may be and still be honored, to allow for clock drift between instances."
+ (Duration/ofMinutes 1))
+
+(def ^:private max-cookie-length
+ "An ISO-8601 instant is at most ~30 characters; anything longer is not one of ours."
+ 40)
+
+(def ^:private last-known-invalidation
+ "The newest invalidation this instance has seen or emitted, as an `Instant`. Any cookie at or before it is ignored,
+ so a given cookie value can trigger a local clear at most once."
+ (atom nil))
+
+(defn- parse-cookie-timestamp
+ "Parse a cookie value as an `Instant`. Returns nil for anything that is not a well-formed, non-future timestamp."
+ ^Instant [value]
+ (when (and (string? value)
+ (<= (count value) max-cookie-length))
+ (try
+ (let [instant (Instant/parse ^String value)]
+ (when-not (.isAfter instant (.plus (Instant/now) max-clock-skew))
+ instant))
+ (catch DateTimeParseException _e
+ (log/debug "Ignoring malformed premium features cookie value")
+ nil))))
+
+(defn- advance-last-known-invalidation!
+ "Atomically move [[last-known-invalidation]] forward to `timestamp` if it is newer. Returns true if it moved, so that
+ of several concurrent callers with the same timestamp exactly one sees true."
+ [^Instant timestamp]
+ (let [[old new] (swap-vals! last-known-invalidation
+ (fn [^Instant last-known]
+ (if (or (nil? last-known) (.isAfter timestamp last-known))
+ timestamp
+ last-known)))]
+ (not= old new)))
(defn- check-and-invalidate-features-cache
- "Check if the cookie timestamp is newer than our last known invalidation, and if so, clear the cache."
+ "If the request cookie carries a valid timestamp newer than the last invalidation this instance knows about, drop
+ this instance's in-process token caches. Never touches the shared DB cache."
[request]
- (when-let [cookie-timestamp (get-in request [:cookies cookie-name :value])]
- (let [last-known @last-known-invalidation]
- (when (and cookie-timestamp
- (or (nil? last-known)
- (try (pos? (compare cookie-timestamp last-known))
- (catch Exception _e
- (log/info "Strange premium features cookie value")
- false))))
- (log/info "Premium features cookie indicates cache is out of date. Clearing...")
- (premium-features/clear-cache!)
- (reset! last-known-invalidation cookie-timestamp)
- ::invalidated))))
+ (when-let [cookie-timestamp (parse-cookie-timestamp (get-in request [:cookies cookie-name :value]))]
+ (when (advance-last-known-invalidation! cookie-timestamp)
+ (log/info "Premium features cookie indicates local cache is out of date. Clearing...")
+ (premium-features/clear-local-cache!))))
(defn- maybe-set-premium-features-cookie
- "Set the premium features last-updated cookie if this request triggered a refresh or propagated an invalidation."
- [invalidated? response]
- (if (or invalidated?
- (-> response :mb/cookies :cookie/premium-features-cache-timestamp))
- (let [timestamp (str (java.time.Instant/now))]
- (reset! last-known-invalidation timestamp)
+ "Set the premium features last-updated cookie if this request refreshed the token status."
+ [response]
+ (if (-> response :mb/cookies :cookie/premium-features-cache-timestamp)
+ (let [timestamp (Instant/now)]
+ (advance-last-known-invalidation! timestamp)
(response/set-cookie (update response :mb/cookies dissoc :cookie/premium-features-cache-timestamp)
cookie-name
- timestamp
+ (str timestamp)
{:path "/"
:max-age (* 5 60)
:same-site :lax}))
response))
(defn wrap-premium-features-cache-check
- "Middleware that checks if the premium features cache needs to be invalidated based on a cookie.
+ "Middleware that checks if the local premium features cache needs to be invalidated based on a cookie.
When premium features are refreshed on one instance, the API sets a cookie with the current timestamp.
This middleware checks that cookie on subsequent requests and clears the local cache if needed,
ensuring consistency in multi-instance deployments."
[handler]
(fn [request respond raise]
- (let [invalidated? (check-and-invalidate-features-cache request)]
- (handler request (comp respond #(maybe-set-premium-features-cookie invalidated? %)) raise))))
+ (check-and-invalidate-features-cache request)
+ (handler request (comp respond maybe-set-premium-features-cookie) raise)))
diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.clj
index 43b048d0760..c9a9291fd63 100644
--- a/test/metabase/premium_features/token_check_test.clj
+++ b/test/metabase/premium_features/token_check_test.clj
@@ -8,6 +8,7 @@
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.core :as mdb]
[metabase.premium-features.core :as premium-features]
+ [metabase.premium-features.db :as premium-features.db]
[metabase.premium-features.task.clear-token-cache]
[metabase.premium-features.test-util :as tu]
[metabase.premium-features.token-check :as token-check]
@@ -163,7 +164,8 @@
(-check-token [_ _]
(swap! call-count inc)
(Thread/sleep 5000))
- (-clear-cache! [_]))
+ (-clear-cache! [_])
+ (-clear-local-cache! [_]))
:circuit-breaker {:failure-threshold-ratio-in-period [10 10 1000]
:delay-ms 50
:success-threshold 1}
@@ -298,7 +300,8 @@
(-check-token [_ _]
(swap! call-count inc)
{:valid true :status "fake" :features ["sso"]})
- (-clear-cache! [_]))
+ (-clear-cache! [_])
+ (-clear-local-cache! [_]))
:soft-ttl (t/hours 12)
:hard-ttl (t/hours 36)}))]
(try
@@ -352,7 +355,8 @@
{:base (reify token-check/TokenChecker
(-check-token [_ token]
(token-response token))
- (-clear-cache! [_]))
+ (-clear-cache! [_])
+ (-clear-local-cache! [_]))
:circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]
:delay-ms 50
:success-threshold 1}
@@ -387,7 +391,8 @@
{:base (reify token-check/TokenChecker
(-check-token [_ token]
(token-response token))
- (-clear-cache! [_]))
+ (-clear-cache! [_])
+ (-clear-local-cache! [_]))
:circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]
:delay-ms 5000
:success-threshold 1}
@@ -777,11 +782,55 @@
(token-check/make-checker
{:base (reify token-check/TokenChecker
(-check-token [_ token] (token-response-fn token))
- (-clear-cache! [_]))
+ (-clear-cache! [_])
+ (-clear-local-cache! [_]))
:local-ttl local-ttl
:soft-ttl soft-ttl
:hard-ttl hard-ttl})))
+(deftest clear-local-cache-test
+ (testing "clear-local-cache! drops in-process memoization but leaves the shared DB row alone"
+ (let [token (tu/random-token)
+ token-hash (#'token-check/hash-token token)
+ call-count (atom 0)
+ deletes (atom 0)
+ good-response {:valid true :status "OK" :features ["sandboxes"] :canonical? true}
+ local-cache (atom {})
+ checker (binding [token-check/*customize-checker* true]
+ (token-check/make-checker
+ {:base (reify token-check/TokenChecker
+ (-check-token [_ _token]
+ (swap! call-count inc)
+ good-response)
+ (-clear-cache! [_])
+ (-clear-local-cache! [_]))
+ ;; long enough that nothing expires during the test
+ :local-ttl (t/hours 1)
+ :soft-ttl (t/hours 12)
+ :hard-ttl (t/hours 36)
+ :db-hash-local-cache local-cache}))]
+ (try
+ (is (= good-response (token-check/check-token checker token)))
+ (is (= 1 @call-count))
+ (mt/with-dynamic-fn-redefs [premium-features.db/delete-token-status-cache! (fn [] (swap! deletes inc))]
+ (token-check/-clear-local-cache! checker))
+ (is (zero? @deletes) "the shared DB row is never deleted")
+ (is (some? (t2/select-one :model/PremiumFeaturesCache :token_hash token-hash)))
+ (is (contains? @local-cache token-hash) "the DB-hash-validated local entry survives")
+ (testing "the next check re-validates against the DB hash and, since it matches, skips the MetaStore"
+ (is (= good-response (token-check/check-token checker token)))
+ (is (= 1 @call-count)))
+ (testing "but a refresh done by another instance (DB hash changed) is picked up"
+ (t2/update! :model/PremiumFeaturesCache :token_hash token-hash {:token_status_hash "refreshed-elsewhere"})
+ (testing "not while the in-process memo still holds"
+ (is (= good-response (token-check/check-token checker token)))
+ (is (= 1 @call-count)))
+ (token-check/-clear-local-cache! checker)
+ (is (= good-response (token-check/check-token checker token)))
+ (is (= 2 @call-count)))
+ (finally
+ (token-check/-clear-cache! checker))))))
+
(deftest db-hash-aware-token-checker-fresh-hit-test
(testing "Fresh cache entry (< soft-ttl) is returned without hitting the underlying checker"
(let [call-count (atom 0)
diff --git a/test/metabase/server/middleware/premium_features_cache_test.clj b/test/metabase/server/middleware/premium_features_cache_test.clj
new file mode 100644
index 00000000000..fe43ccc9bb8
--- /dev/null
+++ b/test/metabase/server/middleware/premium_features_cache_test.clj
@@ -0,0 +1,112 @@
+(ns metabase.server.middleware.premium-features-cache-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.server.middleware.premium-features-cache :as mw.pf-cache]
+ [metabase.test :as mt])
+ (:import
+ (java.time Duration Instant)
+ (java.util.concurrent CountDownLatch)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private cookie-name (var-get #'mw.pf-cache/cookie-name))
+
+(defn- request-with-cookie [value]
+ {:cookies {cookie-name {:value value}}})
+
+(defn- run-middleware
+ "Run the middleware over `request` with a handler that responds with `response`. Returns the response the middleware
+ passed to `respond`."
+ [request response]
+ (let [result (promise)
+ handler (mw.pf-cache/wrap-premium-features-cache-check
+ (fn [_request respond _raise] (respond response)))]
+ (handler request result (fn [e] (throw e)))
+ @result))
+
+(defn- do-with-clear-counter!
+ "Call `(f calls)` with `clear-local-cache!` replaced by a counter and a fresh last-known invalidation. `calls` is the
+ counter atom."
+ [f]
+ (let [calls (atom 0)]
+ (with-redefs [mw.pf-cache/last-known-invalidation (atom nil)]
+ (mt/with-dynamic-fn-redefs [premium-features/clear-local-cache! (fn [] (swap! calls inc))]
+ (f calls)))))
+
+(deftest valid-cookie-clears-local-cache-once-test
+ (do-with-clear-counter!
+ (fn [calls]
+ (let [request (request-with-cookie (str (Instant/now)))]
+ (testing "a valid cookie clears the local cache"
+ (run-middleware request {:status 200})
+ (is (= 1 @calls)))
+ (testing "the same cookie value does not clear the cache again"
+ (run-middleware request {:status 200})
+ (is (= 1 @calls)))))))
+
+(deftest newer-and-older-cookies-test
+ (do-with-clear-counter!
+ (fn [calls]
+ (let [now (Instant/now)]
+ (run-middleware (request-with-cookie (str now)) {:status 200})
+ (is (= 1 @calls))
+ (testing "an older cookie is ignored"
+ (run-middleware (request-with-cookie (str (.minusSeconds now 10))) {:status 200})
+ (is (= 1 @calls)))
+ (testing "a newer cookie clears the cache again"
+ (run-middleware (request-with-cookie (str (.plusSeconds now 1))) {:status 200})
+ (is (= 2 @calls)))))))
+
+(deftest invalid-cookies-are-ignored-test
+ (doseq [[description value] {"no cookie" nil
+ "empty string" ""
+ "malformed value" "not-a-timestamp"
+ "non-ISO timestamp" "2026-01-01 00:00:00.0+00"
+ "far-future value" (str (.plus (Instant/now) (Duration/ofDays 1)))
+ "over-long value" (str (Instant/now) (apply str (repeat 40 "0")))}]
+ (testing description
+ (do-with-clear-counter!
+ (fn [calls]
+ (let [request (if (some? value) (request-with-cookie value) {})
+ response (run-middleware request {:status 200})]
+ (is (zero? @calls) "the local cache is not cleared")
+ (is (= {:status 200} response) "the request still goes through")))))))
+
+(deftest small-clock-skew-is-allowed-test
+ (do-with-clear-counter!
+ (fn [calls]
+ (run-middleware (request-with-cookie (str (.plusSeconds (Instant/now) 30))) {:status 200})
+ (is (= 1 @calls) "a cookie a little in the future (within the allowed clock skew) is honored"))))
+
+(deftest sets-cookie-on-refresh-test
+ (do-with-clear-counter!
+ (fn [calls]
+ (testing "a response without the refresh marker is passed through unchanged"
+ (is (= {:status 200} (run-middleware {} {:status 200}))))
+ (testing "a response with the refresh marker gets the cookie, and the marker is removed"
+ (let [response (run-middleware {} {:status 200
+ :mb/cookies {:cookie/premium-features-cache-timestamp true}})
+ cookie (get-in response [:cookies cookie-name])]
+ (is (some? cookie))
+ (is (instance? Instant (Instant/parse (:value cookie))))
+ (is (=? {:path "/" :max-age 300 :same-site :lax} cookie))
+ (is (not (contains? (:mb/cookies response) :cookie/premium-features-cache-timestamp)))
+ (testing "the instance that set the cookie does not clear its own cache when it sees the cookie"
+ (run-middleware (request-with-cookie (:value cookie)) {:status 200})
+ (is (zero? @calls))))))))
+
+(deftest concurrent-requests-clear-once-test
+ (testing "many concurrent requests with the same new cookie clear the local cache only once"
+ (dotimes [_ 20]
+ (do-with-clear-counter!
+ (fn [calls]
+ (let [start (CountDownLatch. 1)
+ request (request-with-cookie (str (Instant/now)))
+ futures (doall (for [_ (range 32)]
+ (future
+ (.await start)
+ (#'mw.pf-cache/check-and-invalidate-features-cache request))))]
+ (.countDown start)
+ (run! deref futures)
+ (is (= 1 @calls))))))))