Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Token grace period (#64938)

dpsutton7b8edfbad4ae
diff --git a/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.clj b/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.clj
index 4ef628f119d..cf045c9d6b7 100644
--- a/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.clj
@@ -44,7 +44,7 @@
(let [add-on {:product-type product-type}]
(events/publish-event! :event/cloud-add-on-purchase {:details {:add-on add-on}, :user-id api/*current-user-id*})
(hm.client/call :change-add-ons :upsert-add-ons [add-on]))
- (premium-features/clear-cache)
+ (premium-features/clear-cache!)
{:status 200 :body {}}
(catch Exception e
(log/warnf e "Error adding purchasing add-on '%s'" product-type)
diff --git a/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj b/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj
index c9c08248bc6..2af954630f9 100644
--- a/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj
+++ b/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj
@@ -46,7 +46,3 @@
(defenterprise decode-airgap-token
"Decodes the airgap token and returns the decoded token."
:feature :none [token] (decode-token token))
-
-(defenterprise token-valid-now?
- "Check that the decoded token is still valid and returns the decoded token."
- :feature :none [token-status] (valid-now? token-status))
diff --git a/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.clj b/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.clj
index ba528f0998f..30f9c508467 100644
--- a/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.clj
@@ -55,9 +55,9 @@
{:terms_of_service true})))))
(testing "succeeds"
(let [{store-api-proxy :proxy store-api-calls :calls} (semantic.tu/spy (constantly nil))
- {clear-token-cache-proxy :proxy clear-token-cache-calls :calls} (semantic.tu/spy premium-features/clear-cache)]
- (with-redefs [hm.client/call store-api-proxy
- premium-features/clear-cache clear-token-cache-proxy]
+ {clear-token-cache-proxy :proxy clear-token-cache-calls :calls} (semantic.tu/spy premium-features/clear-cache!)]
+ (with-redefs [hm.client/call store-api-proxy
+ premium-features/clear-cache! clear-token-cache-proxy]
(is (=? {}
(mt/user-http-request user :post 200 "ee/cloud-add-ons/metabase-ai"
{:terms_of_service true})))
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index fabb7fbc3c9..936f99eaecb 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -30,7 +30,7 @@
plan-alias
quotas
TokenStatus
- clear-cache]
+ clear-cache!]
(metabase.premium-features.settings
active-users-count
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index 145767cc40d..6ec1ddb9b48 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -26,8 +26,12 @@
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[metabase.util.string :as u.str]
+ [potemkin.types :as p]
[toucan2.connection :as t2.conn]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (com.google.common.cache CacheBuilder RemovalCause RemovalNotification)
+ (java.util.concurrent TimeUnit)))
(set! *warn-on-reflection* true)
@@ -56,10 +60,6 @@
(str/replace #"/$" "")))
"https://token-check.metabase.com"))
-(def store-url
- "Store URL, used as a fallback for token checks and for fetching the list of cloud gateway IPs."
- "https://store.metabase.com")
-
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | TOKEN VALIDATION |
;;; +----------------------------------------------------------------------------------------------------------------+
@@ -151,80 +151,29 @@
[:email :string]]]]]
[:quotas {:optional true} [:sequential [:map]]]])
-(def ^:private ^:const token-status-cache-ttl
- "Amount of time in ms to cache the status of a valid enterprise token before forcing a re-check."
- (u/hours->ms 12))
-
-(def ^{:arglists '([token base-url site-uuid])} ^:private fetch-token-and-parse-body*
- "Caches successful and 4XX API responses for 24 hours. 5XX errors, timeouts, etc. may be transient and will NOT be
- cached, but may trigger the *store-circuit-breaker*."
- (memoize/ttl
- ^{::memoize/args-fn (fn [[token base-url site-uuid]]
- [token base-url site-uuid])}
- (fn [token base-url site-uuid]
- (log/infof "Checking with the MetaStore to see whether token '%s' is valid..." (u.str/mask token))
- (let [{:keys [body status] :as resp} (some-> (token-status-url token base-url)
- (http/get {:query-params (merge (stats-for-token-request)
- {:site-uuid site-uuid
- :mb-version (:tag config/mb-version-info)})
- :throw-exceptions false}))]
- (cond
- (http/success? resp) (some-> body json/decode+kw)
-
- (<= 400 status 499) (some-> body json/decode+kw)
-
- ;; exceptions are not cached.
- :else (throw (ex-info "An unknown error occurred when validating token." {:status status
- :body body})))))
-
- :ttl/threshold token-status-cache-ttl))
-
-(def ^:private store-circuit-breaker-config
- {;; if 10 requests within 10 seconds fail, open the circuit breaker.
- ;; (a lower threshold ratio wouldn't make sense here because successful results are cached, so as soon as we get
- ;; one successful response we're guaranteed to only get successes until cache expiration)
- :failure-threshold-ratio-in-period [10 10 (u/seconds->ms 10)]
- ;; after the circuit is opened, wait 30 seconds before making any more requests to the store
- :delay-ms (u/seconds->ms 30)
- ;; when the circuit breaker is half-open, one request will be permitted. if it's successful, return to normal.
- ;; otherwise we'll wait another 30 seconds.
- :success-threshold 1})
-
-(def ^:dynamic *store-circuit-breaker*
- "A circuit breaker that short-circuits when requests to the API have repeatedly failed.
-
- This prevents a pathological scenario where the store has a temporary outage (long enough for the cache to expire)
- and then all instances everywhere fire off constant requests to get token status. Instead, execution will constantly
- fail instantly until the circuit breaker is closed."
- (dh.cb/circuit-breaker store-circuit-breaker-config))
-
-(def ^:private ^:const fetch-token-status-timeout-ms (u/seconds->ms 10))
+(defn- http-fetch
+ [base-url token site-uuid]
+ (some-> (token-status-url token base-url)
+ (http/get {:query-params (merge (stats-for-token-request)
+ {:site-uuid site-uuid
+ :mb-version (:tag config/mb-version-info)})
+ :throw-exceptions false})))
(defn- fetch-token-and-parse-body
[token base-url site-uuid]
- (try
- (dh/with-circuit-breaker *store-circuit-breaker*
- (dh/with-timeout {:timeout-ms fetch-token-status-timeout-ms
- :interrupt? true}
- (try (fetch-token-and-parse-body* token base-url site-uuid)
- (catch Exception e
- (throw e)))))
- (catch dev.failsafe.TimeoutExceededException _e
- {:valid false
- :status (tru "Unable to validate token")
- :error-details (tru "Token validation timed out.")})
- (catch dev.failsafe.CircuitBreakerOpenException _e
- {:valid false
- :status (tru "Unable to validate token")
- :error-details (tru "Token validation is currently unavailable.")})
- ;; other exceptions are wrapped by Diehard in a FailsafeException. Unwrap them before rethrowing.
- (catch dev.failsafe.FailsafeException e
- (throw (.getCause e)))))
-
-(defn clear-cache
- "Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."
- []
- (memoize/memo-clear! fetch-token-and-parse-body*))
+ (log/infof "Checking with the MetaStore to see whether token '%s' is valid..." (u.str/mask token))
+ (let [{:keys [body status] :as resp} (http-fetch base-url token site-uuid)]
+ (cond
+ (http/success? resp) (some-> body json/decode+kw)
+ ;; todo: what happens if there's no response here? probably should or here
+ (<= 400 status 499) (or (some-> body json/decode+kw)
+ {:valid false
+ :status "Unable to validate token"
+ :error-details "Token validation provided no response"})
+
+ ;; exceptions are not cached.
+ :else (throw (ex-info "An unknown error occurred when validating token." {:status status
+ :body body})))))
;;;;;;;;;;;;;;;;;;;; Airgap Tokens ;;;;;;;;;;;;;;;;;;;;
@@ -245,33 +194,16 @@
(when (> (t2/count :model/User :is_active true, :type :personal) max-users)
(throw (Exception. (trs "You have reached the maximum number of users ({0}) for your plan. Please upgrade to add more users." max-users))))))
-(mu/defn- fetch-token-status* :- TokenStatus
- "Fetch info about the validity of `token` from the MetaStore."
+(mu/defn- decode-token* :- TokenStatus
+ "Decode a token. If you get a positive response about the token, even if it is not valid, return that. Errors will
+ be caught further up with appropriate fall backs, retry strategies, and grace periods for features."
[token :- TokenStr]
;; NB that we fetch any settings from this thread, not inside on of the futures in the inner fetch calls. We
;; will have taken a lock to call through to here, and could create a deadlock with the future's thread. See
;; https://github.com/metabase/metabase/pull/38029/
(cond (mr/validate [:re RemoteCheckedToken] token)
- ;; attempt to query the metastore API about the status of this token. If the request doesn't complete in a
- ;; reasonable amount of time throw a timeout exception
(let [site-uuid (premium-features.settings/site-uuid-for-premium-features-token-checks)]
- (try (fetch-token-and-parse-body token token-check-url site-uuid)
- (catch Exception e1
- (log/errorf e1 "Error fetching token status from %s:" token-check-url)
- ;; Try the fallback URL, which was the default URL prior to 45.2
- (try (fetch-token-and-parse-body token store-url site-uuid)
- ;; if there was an error fetching the token from both the normal and fallback URLs, log the
- ;; first error and return a generic message about the token being invalid. This message
- ;; will get displayed in the Settings page in the admin panel so we do not want something
- ;; complicated
- (catch Exception e2
- (log/errorf e2 "Error fetching token status from %s:" store-url)
- (let [body (u/ignore-exceptions (some-> (ex-data e1) :body json/decode+kw))]
- (or
- body
- {:valid false
- :status (tru "Unable to validate token")
- :error-details (.getMessage e1)})))))))
+ (fetch-token-and-parse-body token token-check-url site-uuid))
(mr/validate [:re AirgapToken] token)
(do
@@ -287,40 +219,190 @@
(def ^:dynamic *token-check-happening* "Var to prevent recursive calls to `fetch-token-status`" false)
-(let [lock (Object.)]
- (defn- fetch-token-status
- "Locked version of `fetch-token-status*` allowing one request at a time."
- [token]
- (when *token-check-happening*
- (throw (ex-info "Token check is being called recursively, there is a good chance some `defenterprise` is causing this"
- {:pass-thru true})))
- (locking lock
- (binding [*token-check-happening* true]
- (fetch-token-status* token)))))
-
-(declare token-valid-now?)
-
-(mu/defn- valid-token->features :- [:set ms/NonBlankString]
- [token :- TokenStr]
- (assert ((requiring-resolve 'metabase.app-db.core/db-is-set-up?)) "Metabase DB is not yet set up")
- (let [{:keys [valid status features error-details] :as token-status} (fetch-token-status token)]
- ;; if token isn't valid throw an Exception with the `:status` message
- (when-not valid
- (throw (ex-info status
- {:status-code 400,
- :error-details error-details})))
- (when (and (mr/validate [:re AirgapToken] token)
- (not (token-valid-now? token-status)))
- (throw (ex-info status
- {:status-code 400
- :error-details (tru "Airgapped token is no longer valid. Please contact Metabase support.")})))
- ;; otherwise return the features this token supports
- (set features)))
+(p/defprotocol+ GracePeriod
+ "A protocol for providing a grace period for token features in the event they are not fetchable for a little while."
+ (save! [_ token features] "Save the features for a particular token.")
+ (retrieve [_ token] "Attempt to retrieve features associated with a token. This is best effort, perhaps never set,
+ perhaps has timed out. Possible this is nil."))
+
+(defn guava-cache-grace-period
+ "Create a grace period of n units. This is just an expiring map using a guava cache. Note this is not sensitive to read times but to write times.
+
+
+ (let [grace (guava-cache-grace-period 20 TimeUnit/MILLISECONDS)]
+ (save! grace \"token\" #{\"features\"})
+ (println \"found tokens?: \"
+ (if (= (retrieve grace \"token\") #{\"features\"})
+ \"🟒\"
+ \"πŸ”΄\"))
+ (Thread/sleep 40)
+ (println \"expecting not to find tokens?: \"
+ (if (retrieve grace \"token\")
+ \"πŸ”΄\"
+ \"🟒\")))
+ found tokens?: 🟒
+ expecting not to find tokens?: 🟒
+ nil"
+ [^long n ^TimeUnit units]
+ (let [guava-cache (.. (CacheBuilder/newBuilder)
+ (expireAfterWrite n units)
+ (removalListener (fn [^RemovalNotification rn]
+ (let [cause (.getCause rn)]
+ (when (= RemovalCause/EXPIRED cause)
+ (log/warnf "Removing token: %s from grace period cache"
+ (u.str/mask (.getKey rn)))))))
+ (build))]
+ (reify GracePeriod
+ (save! [_ token features] (.put guava-cache token features))
+ (retrieve [_ token]
+ (when token
+ (let [value (.get guava-cache token (constantly ::not-present))]
+ (when-not (identical? value ::not-present)
+ value)))))))
+
+(p/defprotocol+ TokenChecker
+ "Protocol for checking tokens with cache management."
+ (-check-token [this token]
+ "Check a token and return TokenStatus map. May throw exceptions on failure.")
+ (-clear-cache! [this]
+ "Clear any caches in this checker and any wrapped checkers.
+ Returns nil. Implementations should delegate to wrapped checkers."))
+
+(def store-and-airgap-token-checker
+ "Creates a basic token checker that handles HTTP requests and airgap tokens.
+ No caching, no retries, no grace period - just the core logic."
+ (reify TokenChecker
+ (-check-token [_ token]
+ (decode-token* token))
+ (-clear-cache! [_]
+ ;; No cache to clear at this level
+ nil)))
+
+(defn circuit-breaker-token-checker
+ "Wraps a token checker with circuit breaker and timeout logic."
+ [token-checker {:keys [circuit-breaker timeout-ms lock]
+ :or {lock (Object.)}}]
+ (let [breaker (dh.cb/circuit-breaker circuit-breaker)]
+ (reify TokenChecker
+ (-check-token [_ token]
+ (when *token-check-happening*
+ (throw (ex-info "Token check is being called recursively, there is a good chance some `defenterprise` is causing this"
+ {:pass-thru true})))
+ (locking lock
+ (binding [*token-check-happening* true]
+ (try (dh/with-circuit-breaker breaker
+ (dh/with-timeout {:timeout-ms timeout-ms
+ :interrupt? true}
+ (-check-token token-checker token)))
+ (catch dev.failsafe.CircuitBreakerOpenException _e
+ (throw (ex-info (tru "Token validation is currently unavailable.")
+ {:cause :circuit-breaker})))
+ ;; other exceptions are wrapped by Diehard in a FailsafeException. Unwrap them before
+ ;; rethrowing.
+ (catch dev.failsafe.FailsafeException e
+ (throw (.getCause e)))))))
+ (-clear-cache! [_]
+ ;; No cache at this level, but delegate to wrapped checker
+ (-clear-cache! token-checker)))))
+
+(defn cached-token-checker
+ "Wraps a token checker with TTL-based memoization."
+ [token-checker {:keys [ttl-ms]}]
+ (let [cached-check (memoize/ttl
+ (fn [token] (-check-token token-checker token))
+ :ttl/threshold ttl-ms)]
+ (reify TokenChecker
+ (-check-token [_ token]
+ (cached-check token))
+ (-clear-cache! [_]
+ ;; Clear THIS layer's cache
+ (memoize/memo-clear! cached-check)
+ ;; AND delegate to wrapped checker
+ (-clear-cache! token-checker)))))
+
+(defn grace-period-token-checker
+ "Wraps a token checker with grace period fallback logic."
+ [token-checker {:keys [grace-period]}]
+ (let [periodic-logger (memoize/ttl
+ (fn [_token] (log/info "Using token from grace period"))
+ :ttl/threshold (u/hours->ms 4))]
+ (reify TokenChecker
+ (-check-token [_ token]
+ (try (let [response (-check-token token-checker token)]
+ (save! grace-period token response)
+ response)
+ (catch Exception e
+ (or (when-let [grace (retrieve grace-period token)]
+ (periodic-logger token)
+ grace)
+ (throw e)))))
+ (-clear-cache! [_]
+ ;; The grace period itself doesn't need clearing (it expires naturally)
+ ;; but we should clear the periodic logger cache
+ (memoize/memo-clear! periodic-logger)
+ ;; AND delegate to wrapped checker
+ (-clear-cache! token-checker)))))
+
+(defn- error-catching-token-checker
+ [token-checker]
+ (reify TokenChecker
+ (-check-token [_ token]
+ (try (-check-token token-checker token)
+ (catch Exception e
+ (u/ignore-exceptions (some-> (ex-data e) :body json/decode+kw))
+ {:valid false
+ :status (tru "Unable to validate token")
+ :error-details (.getMessage e)})))
+ (-clear-cache! [_] (-clear-cache! token-checker))))
+
+(def ^:dynamic *customize-checker*
+ "Dynamic variable allowing for customized token checkers. In the app, we want all of these in place. Only in tests
+ should we construct ones without circuit breakers "
+ false)
-(defn -token-status
- "Getter for the [[metabase.premium-features.settings/token-status]] setting."
+(defn make-checker
+ "Make a token checker. Takes a base [[TokenChecker]] token checker and then arguments for the middleware-style
+ wrapping [[TokenChecker]] arguments. "
+ [{:keys [base circuit-breaker timeout-ms ttl-ms grace-period]
+ :or {base store-and-airgap-token-checker}}]
+ (when-not *customize-checker*
+ (assert (and base circuit-breaker timeout-ms ttl-ms grace-period)
+ "Must provide all arguments for token checker"))
+ (cond-> base
+ ;; don't do it too much
+ circuit-breaker
+ (circuit-breaker-token-checker {:circuit-breaker circuit-breaker
+ :timeout-ms timeout-ms})
+ ;; hold onto results for a while
+ ttl-ms
+ (cached-token-checker {:ttl-ms ttl-ms})
+ ;; in case of errors, if we have a recent response use it
+ grace-period
+ (grace-period-token-checker {:grace-period grace-period})
+ :always
+ (error-catching-token-checker)))
+
+(def token-checker
+ "The token checker. Combines http/airgapping vaildation, circuit breaking, grace periods, caching, and error
+ handling."
+ (make-checker {:base store-and-airgap-token-checker
+ :circuit-breaker {:failure-threshold-ratio-in-period [10 10 (u/seconds->ms 10)]
+ :delay-ms (u/seconds->ms 30)
+ :success-threshold 1}
+ :timeout-ms (u/seconds->ms 10)
+ :ttl-ms (u/hours->ms 12)
+ :grace-period (guava-cache-grace-period 36 TimeUnit/HOURS)}))
+
+(defn clear-cache!
+ "Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."
[]
- (some-> (premium-features.settings/premium-embedding-token) (fetch-token-status)))
+ (-clear-cache! token-checker))
+
+(defn check-token
+ "Public entrypoint to the token checking."
+ ([token] (check-token token-checker token))
+ ([checker token]
+ (-check-token checker token)))
(defn -set-premium-embedding-token!
"Setter for the [[metabase.premium-features.settings/token-status]] setting."
@@ -334,7 +416,9 @@
(mr/validate [:re AirgapToken] new-value))
(throw (ex-info (tru "Token format is invalid.")
{:status-code 400, :error-details "Token should be 64 hexadecimal characters."})))
- (valid-token->features new-value)
+ (let [decoded (check-token new-value)]
+ (when-not (:valid decoded)
+ (throw (ex-info "Invalid token" {:token (u.str/mask new-value)}))))
(log/info "Token is valid."))
(setting/set-value-of-type! :string :premium-embedding-token new-value)
(catch Throwable e
@@ -360,7 +444,9 @@
"Get the features associated with the system's premium features token."
[]
(try
- (or (some-> (premium-features.settings/premium-embedding-token) valid-token->features)
+ (or (some-> (premium-features.settings/premium-embedding-token)
+ (check-token)
+ :features set)
#{})
(catch Throwable e
(when (:pass-thru (ex-data e))
@@ -368,19 +454,25 @@
(cached-logger (premium-features.settings/premium-embedding-token) e)
#{}))))
+(defn -token-status
+ "Getter for the [[metabase.premium-features.settings/token-status]] setting."
+ []
+ (some-> (premium-features.settings/premium-embedding-token)
+ (check-token)))
+
(mu/defn plan-alias :- [:maybe :string]
"Returns a string representing the instance's current plan, if included in the last token status request."
[]
(some-> (premium-features.settings/premium-embedding-token)
- fetch-token-status
+ (check-token)
:plan-alias))
(mu/defn quotas :- [:maybe [:sequential [:map]]]
"Returns a vector of maps for each quota of the subscription."
[]
- (memoize/memo-clear! fetch-token-and-parse-body*)
+ (clear-cache!)
(some-> (premium-features.settings/premium-embedding-token)
- fetch-token-status
+ (check-token)
:quotas))
(defn has-any-features?
@@ -430,9 +522,3 @@
metabase-enterprise.premium-features.airgap
[_]
{})
-
-(defenterprise token-valid-now?
- "In OSS, this returns false."
- metabase-enterprise.premium-features.airgap
- [_]
- false)
diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.clj
index 96bc413fad0..ff6c2d20709 100644
--- a/test/metabase/premium_features/token_check_test.clj
+++ b/test/metabase/premium_features/token_check_test.clj
@@ -1,93 +1,33 @@
(ns metabase.premium-features.token-check-test
(:require
[clj-http.client :as http]
- [clj-http.fake :as http-fake]
[clojure.test :refer :all]
- [diehard.circuit-breaker :as dh.cb]
[mb.hawk.parallel]
[metabase.app-db.connection :as mdb.connection]
- [metabase.config.core :as config]
[metabase.premium-features.core :as premium-features]
- [metabase.premium-features.settings :as premium-features.settings]
[metabase.premium-features.test-util :as tu]
[metabase.premium-features.token-check :as token-check]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
- [metabase.util.json :as json]
[metabase.util.malli.registry :as mr]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.util.concurrent TimeUnit)))
(set! *warn-on-reflection* true)
-(defn- open-circuit-breaker! [cb]
- (.open ^dev.failsafe.CircuitBreaker cb))
-
-(defmacro with-open-circuit-breaker! [& body]
- `(binding [token-check/*store-circuit-breaker* (dh.cb/circuit-breaker
- @#'token-check/store-circuit-breaker-config)]
- (open-circuit-breaker! token-check/*store-circuit-breaker*)
- (do ~@body)))
-
-(defn reset-circuit-breaker-fixture [f]
- (binding [token-check/*store-circuit-breaker* (dh.cb/circuit-breaker
- @#'token-check/store-circuit-breaker-config)]
- (f)))
-
(use-fixtures :once (fixtures/initialize :db))
-(use-fixtures :each reset-circuit-breaker-fixture)
-(defn- token-status-response
- [token token-check-response]
- (http-fake/with-fake-routes-in-isolation
- {{:address (#'token-check/token-status-url token @#'token-check/token-check-url)
- :query-params (merge (#'token-check/stats-for-token-request)
- {:site-uuid (premium-features.settings/site-uuid-for-premium-features-token-checks)
- :mb-version (:tag config/mb-version-info)})}
- (constantly token-check-response)}
- (#'token-check/fetch-token-status* token)))
-
-(def ^:private token-response-fixture
- (json/encode {:valid true
- :status "fake"
- :features ["test" "fixture"]
- :trial false}))
-
-(deftest ^:parallel fetch-token-status-test
+(deftest log-tests
(let [token (tu/random-token)
print-token (apply str (concat (take 4 token) "..." (take-last 4 token)))]
(testing "Do not log the token (#18249)"
(mt/with-log-messages-for-level [messages :info]
- (#'token-check/fetch-token-status* token)
+ (token-check/check-token token)
(let [logs (mapv :message (messages))]
(is (every? (complement #(re-find (re-pattern token) %)) logs))
(is (= 1 (count (filter #(re-find (re-pattern print-token) %) logs)))))))))
-(deftest ^:parallel fetch-token-status-test-2
- (testing "With the backend unavailable"
- (let [result (token-status-response (tu/random-token) {:status 500})]
- (is (false? (:valid result))))))
-
-(deftest ^:parallel fetch-token-status-test-3
- (testing "On other errors"
- (binding [http/request (fn [& _]
- ;; note originally the code caught clojure.lang.ExceptionInfo so don't
- ;; throw an ex-info here
- (throw (Exception. "network issues")))]
- (is (= {:valid false
- :status "Unable to validate token"
- :error-details "network issues"}
- (#'token-check/fetch-token-status (apply str (repeat 64 "b"))))))))
-
-(deftest fetch-token-caches-successful-responses
- (testing "For successful responses, the result is cached"
- (let [call-count (atom 0)
- token (tu/random-token)]
- (binding [http/request (fn [& _]
- (swap! call-count inc)
- {:status 200 :body "{\"valid\": true, \"status\": \"fake\"}"})]
- (dotimes [_ 10] (#'token-check/fetch-token-status token))
- (is (= 1 @call-count))))))
-
(deftest fetch-token-caches-invalid-responses
(testing "For 4XX responses, the result is cached"
(let [call-count (atom 0)
@@ -95,91 +35,183 @@
(binding [http/request (fn [& _]
(swap! call-count inc)
{:status 400 :body "{\"valid\": false, \"status\": \"fake\"}"})]
- (dotimes [_ 10] (#'token-check/fetch-token-status token))
+ (token-check/-clear-cache! token-check/token-checker)
+ (dotimes [_ 10] (token-check/check-token token))
(is (= 1 @call-count))))))
(deftest fetch-token-does-not-cache-exceptions
- (testing "For timeouts, 5XX errors, etc. we don't cache the result"
- (let [call-count (atom 0)
- token (tu/random-token)]
- (binding [http/request (fn [& _]
- (swap! call-count inc)
- (throw (ex-info "oh, fiddlesticks" {})))]
- (dotimes [_ 5] (#'token-check/fetch-token-status token))
- ;; Note that we have a fallback URL that gets hit in this case (see
- ;; https://github.com/metabase/metabase/issues/27036) and 2x5=10
- (is (= 10 @call-count))))))
-
-(deftest fetch-token-does-not-cache-5XX-responses
(let [call-count (atom 0)
- token (tu/random-token)]
- (binding [http/request (fn [& _]
- (swap! call-count inc)
- {:status 500})]
- (dotimes [_ 10] (#'token-check/fetch-token-status token))
- ;; Same as above, we have a fallback URL that gets hit in this case (see
- ;; https://github.com/metabase/metabase/issues/27036) and 2x10=20
- (is (= 10 @call-count)))))
-
-(deftest fetch-token-is-circuit-broken
- (let [call-count (atom 0)]
- (with-open-circuit-breaker!
- (binding [http/request (fn [& _] (swap! call-count inc))]
- (is (= {:valid false
- :status "Unable to validate token"
- :error-details "Token validation is currently unavailable."}
- (#'token-check/fetch-token-status (tu/random-token))))
- (is (= 0 @call-count))))))
-
-(deftest ^:parallel fetch-token-status-test-4
- (testing "With a valid token"
- (let [result (token-status-response (tu/random-token) {:status 200
- :body token-response-fixture})]
- (is (:valid result))
- (is (contains? (set (:features result)) "test")))))
+ token (tu/random-token)
+ response (atom :error)
+ ;; i've removed the circuit breaker from the stack. that sometimes shuts down the tests in a way we don't want
+ ;; to exercise here
+ checker (binding [token-check/*customize-checker* true]
+ (token-check/make-checker {:ttl-ms 500
+ :grace-period (token-check/guava-cache-grace-period 36 TimeUnit/HOURS)}))]
+ (with-redefs [token-check/http-fetch (fn [& _]
+ (swap! call-count inc)
+ (case @response
+ :error (throw (ex-info "kaboom" {:ka :boom}))
+ :500 {:status 500}))]
+ (testing "For timeouts, 5XX errors, etc. we don't cache the result"
+ (dotimes [_ 5] (token-check/check-token checker token))
+ (is (= 5 @call-count)))
+ (testing "Does not cache 500 responses"
+ (reset! call-count 0)
+ (reset! response :500)
+ (dotimes [_ 5] (token-check/check-token checker token))
+ (is (= 5 @call-count))))))
(deftest not-found-test
(mt/with-log-level :fatal
- ;; `partial=` here in case the Cloud API starts including extra keys... this is a "dangerous" test since changes
- ;; upstream in Cloud could break this. We probably want to catch that stuff anyway tho in tests rather than waiting
- ;; for bug reports to come in
- (is (partial= {:valid false, :status "Token does not exist."}
- (#'token-check/fetch-token-status* (tu/random-token))))))
+ (is (=? {:valid false, :status "Token does not exist."}
+ (token-check/check-token (tu/random-token))))))
(deftest fetch-token-does-not-call-db-when-cached
(testing "No DB calls are made when checking token status if the status is cached"
(let [token (tu/random-token)
- _ (#'token-check/fetch-token-status token)
+ _ (token-check/check-token token)
;; Sigh. This is really quite horrific. But we need some wiggle room here: any endpoint that gets some setting
;; inside it is going to check to see whether it's time for an update check. If it is, it'll hit the DB to see
;; when settings were last updated, and the count will be incremented. Therefore, let's do this a few times...
call-counts (repeatedly 3 (fn []
(t2/with-call-count [call-count]
- (#'token-check/fetch-token-status token)
+ (token-check/check-token token)
(call-count))))]
;; ... and then make sure that *some* of the times, we didn't hit the DB again.
(is (some zero? call-counts)))))
+(deftest token-checker-test
+ (let [token (tu/random-token)
+ behavior (atom :success)
+ call-count (atom 0)
+ good-response {:valid true
+ :status :fake
+ :features [:feature1 :feature2]}
+ no-features {:valid false :status "Unable to validate token" :error-details "network issues!"}
+ token-response (fn [_token]
+ (swap! call-count inc)
+ (case @behavior
+ :success good-response
+ :timeout (Thread/sleep 200)
+ :error (throw (ex-info "network issues!" {:ka :boom}))))
+ checker (token-check/make-checker
+ {:base (reify token-check/TokenChecker
+ (-check-token [_ token]
+ (token-response token))
+ (-clear-cache! [_]))
+ :circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]
+ :delay-ms 50
+ :success-threshold 1}
+ :timeout-ms 100
+ :ttl-ms 200
+ :grace-period (token-check/guava-cache-grace-period 1000 TimeUnit/MILLISECONDS)})]
+ (testing "when no information is present, hits the underlying"
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (= 1 @call-count)))
+ (testing "hitting it again in the same timeperiod does not hit the underlying checker"
+ (is (= good-response (token-check/-check-token checker token)))
+ (is (= 1 @call-count)))
+ (testing "While it errors, we have circuit breaking"
+ (reset! behavior :error)
+ (Thread/sleep 200)
+ (dotimes [_ 1000] (token-check/-check-token checker token))
+ (is (< @call-count 50))
+ (testing "But we always get a good response from the grace period"
+ (is (= good-response (token-check/-check-token checker token)))))
+ (testing "But eventually grace period elapses"
+ (Thread/sleep 1000)
+ (is (= no-features (token-check/-check-token checker token))))
+ (testing "When connectivity is restored we get successful token checks"
+ (reset! behavior :success)
+ (Thread/sleep 100) ;; wait for circuit breaker to open back up
+ (is (= good-response (token-check/-check-token checker token))))))
+
+(deftest grace-period-test
+ (testing "implementation check"
+ (let [token (tu/random-token)
+ behavior (atom :success)
+ success-token {:valid true :status :fake :features [:feature1 :feature2]}
+ restored-token {:valid true :status :fake :features [:new-feature]}
+ invalid {:valid false, :status "Unable to validate token", :error-details nil}
+ underlying (reify token-check/TokenChecker
+ (-check-token [_ t]
+ (when (= t token)
+ (case @behavior
+ :success success-token
+ :error (throw (ex-info "network troubles!" {:ka :boom}))
+ :network-restored restored-token)))
+ (-clear-cache! [_]))
+ grace (binding [token-check/*customize-checker* true]
+ (token-check/make-checker
+ {:base underlying
+ :grace-period (token-check/guava-cache-grace-period 20 TimeUnit/MILLISECONDS)}))]
+ (is (= success-token (token-check/check-token grace token)))
+ (is (= invalid (token-check/check-token grace (tu/random-token))))
+ (testing "During errors"
+ (reset! behavior :error)
+ (is (= success-token (token-check/check-token grace token)))
+ (is (= invalid (token-check/check-token grace (tu/random-token))))
+ (Thread/sleep 40) ;; our simple one here has a grace period of 20 milliseconds
+ (testing "but it expires"
+ (is (= {:valid false :status "Unable to validate token" :error-details "network troubles!"}
+ (token-check/check-token grace token)))))
+ (testing "When good"
+ (reset! behavior :success)
+ (is (= success-token (token-check/check-token grace token))))
+ (testing "We can enter the grace period"
+ (reset! behavior :error)
+ (is (= success-token (token-check/check-token grace token))))
+ (testing "And then we immediately get new token when network restores"
+ (reset! behavior :network-restored)
+ (is (= restored-token (token-check/check-token grace token)))))))
+
+(deftest e2e
+ (testing "token check hits the network"
+ (let [token (tu/random-token)
+ ;; todo: need to check that we use the grace period
+ time-passes! token-check/clear-cache!
+ call-count (atom 0)]
+ (with-redefs [token-check/http-fetch (fn [& _]
+ (swap! call-count inc)
+ {:status 200
+ :body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"})]
+ (= {:valid true, :status "fake", :features ["fake" "features"]}
+ (token-check/check-token token))
+ (is (= 1 @call-count)))
+ (with-redefs [token-check/http-fetch (fn [& _]
+ (swap! call-count inc)
+ (throw (ex-info "network failure!" {})))]
+ (testing "Doesn't hit the network inside of cache duration"
+ (is (= {:valid true, :status "fake", :features ["fake" "features"]}
+ (token-check/check-token token)))
+ (is (= 1 @call-count)))
+ (time-passes!) ;; we can clear the cache but don't have a great way to expire the grace period
+ (testing "Hits the network periodically (12 hours)"
+ (let [response (token-check/check-token token)]
+ ;; called but we got network failure from redefs
+ (is (= 2 @call-count))
+ (testing "Grace period supplements errors"
+ (is (= {:valid true, :status "fake", :features ["fake" "features"]}
+ response)))))))))
+
(deftest token-status-setting-test
(testing "If a `premium-embedding-token` has been set, the `token-status` setting should return the response
from the store.metabase.com endpoint for that token."
- (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]
- (is (= {:valid false, :status "Token does not exist."}
- (premium-features/token-status)))))
+ (is (= {:valid false, :status "Token does not exist."}
+ (token-check/check-token (tu/random-token)))))
(testing "If premium-embedding-token is nil, the token-status setting should also be nil."
(mt/with-temporary-setting-values [premium-embedding-token nil]
(is (nil? (premium-features/token-status))))))
(deftest active-users-count-setting-test
- (mt/with-temp
- [:model/User _ {:is_active false}]
- (testing "returns the number of active users"
- (is (= (t2/count :model/User :is_active true :type :personal)
- (premium-features/active-users-count))))
+ (testing "returns the number of active users"
+ (is (= (t2/count :model/User :is_active true :type :personal)
+ (premium-features/active-users-count))))
- (testing "Default to 0 if db is not setup yet"
- (binding [mdb.connection/*application-db* {:status (atom nil)}]
- (is (zero? (premium-features/active-users-count)))))))
+ (testing "Default to 0 if db is not setup yet"
+ (binding [mdb.connection/*application-db* {:status (atom nil)}]
+ (is (zero? (premium-features/active-users-count))))))
(deftest RemoteCheckedToken-regexp
(testing "valid tokens"
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index c8c5819ceb4..9c3e7193672 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -600,7 +600,7 @@
The token-value binding will contain the random token that was set."
[[token-value] & body]
`(let [~token-value (premium-features.test-util/random-token)]
- (with-redefs [metabase.premium-features.token-check/fetch-token-status
+ (with-redefs [metabase.premium-features.token-check/check-token
(constantly {:valid true
:status "fake"
:features ["test" "fixture"]