The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Token grace period (#64938)
dpsutton7b8edfbad4ae
diff --git a/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.clj b/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.cljindex 4ef628f119d..cf045c9d6b7 100644--- a/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.clj+++ b/enterprise/backend/src/metabase_enterprise/cloud_add_ons/api.clj@@ -44,7 +44,7 @@(let [add-on {:product-type product-type}](events/publish-event! :event/cloud-add-on-purchase {:details {:add-on add-on}, :user-id api/*current-user-id*})(hm.client/call :change-add-ons :upsert-add-ons [add-on]))- (premium-features/clear-cache)+ (premium-features/clear-cache!){:status 200 :body {}}(catch Exception e(log/warnf e "Error adding purchasing add-on '%s'" product-type)diff --git a/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj b/enterprise/backend/src/metabase_enterprise/premium_features/airgap.cljindex c9c08248bc6..2af954630f9 100644--- a/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj+++ b/enterprise/backend/src/metabase_enterprise/premium_features/airgap.clj@@ -46,7 +46,3 @@(defenterprise decode-airgap-token"Decodes the airgap token and returns the decoded token.":feature :none [token] (decode-token token))--(defenterprise token-valid-now?- "Check that the decoded token is still valid and returns the decoded token."- :feature :none [token-status] (valid-now? token-status))diff --git a/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.clj b/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.cljindex ba528f0998f..30f9c508467 100644--- a/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/cloud_add_ons/api_test.clj@@ -55,9 +55,9 @@{:terms_of_service true})))))(testing "succeeds"(let [{store-api-proxy :proxy store-api-calls :calls} (semantic.tu/spy (constantly nil))- {clear-token-cache-proxy :proxy clear-token-cache-calls :calls} (semantic.tu/spy premium-features/clear-cache)]- (with-redefs [hm.client/call store-api-proxy- premium-features/clear-cache clear-token-cache-proxy]+ {clear-token-cache-proxy :proxy clear-token-cache-calls :calls} (semantic.tu/spy premium-features/clear-cache!)]+ (with-redefs [hm.client/call store-api-proxy+ premium-features/clear-cache! clear-token-cache-proxy](is (=? {}(mt/user-http-request user :post 200 "ee/cloud-add-ons/metabase-ai"{:terms_of_service true})))diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.cljindex fabb7fbc3c9..936f99eaecb 100644--- a/src/metabase/premium_features/core.clj+++ b/src/metabase/premium_features/core.clj@@ -30,7 +30,7 @@plan-aliasquotasTokenStatus- clear-cache]+ clear-cache!](metabase.premium-features.settingsactive-users-countdiff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex 145767cc40d..6ec1ddb9b48 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -26,8 +26,12 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.util.string :as u.str]+ [potemkin.types :as p][toucan2.connection :as t2.conn]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (com.google.common.cache CacheBuilder RemovalCause RemovalNotification)+ (java.util.concurrent TimeUnit)))(set! *warn-on-reflection* true)@@ -56,10 +60,6 @@(str/replace #"/$" "")))"https://token-check.metabase.com"))-(def store-url- "Store URL, used as a fallback for token checks and for fetching the list of cloud gateway IPs."- "https://store.metabase.com")-;;; +----------------------------------------------------------------------------------------------------------------+;;; | TOKEN VALIDATION |;;; +----------------------------------------------------------------------------------------------------------------+@@ -151,80 +151,29 @@[:email :string]]]]][:quotas {:optional true} [:sequential [:map]]]])-(def ^:private ^:const token-status-cache-ttl- "Amount of time in ms to cache the status of a valid enterprise token before forcing a re-check."- (u/hours->ms 12))--(def ^{:arglists '([token base-url site-uuid])} ^:private fetch-token-and-parse-body*- "Caches successful and 4XX API responses for 24 hours. 5XX errors, timeouts, etc. may be transient and will NOT be- cached, but may trigger the *store-circuit-breaker*."- (memoize/ttl- ^{::memoize/args-fn (fn [[token base-url site-uuid]]- [token base-url site-uuid])}- (fn [token base-url site-uuid]- (log/infof "Checking with the MetaStore to see whether token '%s' is valid..." (u.str/mask token))- (let [{:keys [body status] :as resp} (some-> (token-status-url token base-url)- (http/get {:query-params (merge (stats-for-token-request)- {:site-uuid site-uuid- :mb-version (:tag config/mb-version-info)})- :throw-exceptions false}))]- (cond- (http/success? resp) (some-> body json/decode+kw)-- (<= 400 status 499) (some-> body json/decode+kw)-- ;; exceptions are not cached.- :else (throw (ex-info "An unknown error occurred when validating token." {:status status- :body body})))))-- :ttl/threshold token-status-cache-ttl))--(def ^:private store-circuit-breaker-config- {;; if 10 requests within 10 seconds fail, open the circuit breaker.- ;; (a lower threshold ratio wouldn't make sense here because successful results are cached, so as soon as we get- ;; one successful response we're guaranteed to only get successes until cache expiration)- :failure-threshold-ratio-in-period [10 10 (u/seconds->ms 10)]- ;; after the circuit is opened, wait 30 seconds before making any more requests to the store- :delay-ms (u/seconds->ms 30)- ;; when the circuit breaker is half-open, one request will be permitted. if it's successful, return to normal.- ;; otherwise we'll wait another 30 seconds.- :success-threshold 1})--(def ^:dynamic *store-circuit-breaker*- "A circuit breaker that short-circuits when requests to the API have repeatedly failed.-- This prevents a pathological scenario where the store has a temporary outage (long enough for the cache to expire)- and then all instances everywhere fire off constant requests to get token status. Instead, execution will constantly- fail instantly until the circuit breaker is closed."- (dh.cb/circuit-breaker store-circuit-breaker-config))--(def ^:private ^:const fetch-token-status-timeout-ms (u/seconds->ms 10))+(defn- http-fetch+ [base-url token site-uuid]+ (some-> (token-status-url token base-url)+ (http/get {:query-params (merge (stats-for-token-request)+ {:site-uuid site-uuid+ :mb-version (:tag config/mb-version-info)})+ :throw-exceptions false})))(defn- fetch-token-and-parse-body[token base-url site-uuid]- (try- (dh/with-circuit-breaker *store-circuit-breaker*- (dh/with-timeout {:timeout-ms fetch-token-status-timeout-ms- :interrupt? true}- (try (fetch-token-and-parse-body* token base-url site-uuid)- (catch Exception e- (throw e)))))- (catch dev.failsafe.TimeoutExceededException _e- {:valid false- :status (tru "Unable to validate token")- :error-details (tru "Token validation timed out.")})- (catch dev.failsafe.CircuitBreakerOpenException _e- {:valid false- :status (tru "Unable to validate token")- :error-details (tru "Token validation is currently unavailable.")})- ;; other exceptions are wrapped by Diehard in a FailsafeException. Unwrap them before rethrowing.- (catch dev.failsafe.FailsafeException e- (throw (.getCause e)))))--(defn clear-cache- "Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."- []- (memoize/memo-clear! fetch-token-and-parse-body*))+ (log/infof "Checking with the MetaStore to see whether token '%s' is valid..." (u.str/mask token))+ (let [{:keys [body status] :as resp} (http-fetch base-url token site-uuid)]+ (cond+ (http/success? resp) (some-> body json/decode+kw)+ ;; todo: what happens if there's no response here? probably should or here+ (<= 400 status 499) (or (some-> body json/decode+kw)+ {:valid false+ :status "Unable to validate token"+ :error-details "Token validation provided no response"})++ ;; exceptions are not cached.+ :else (throw (ex-info "An unknown error occurred when validating token." {:status status+ :body body})))));;;;;;;;;;;;;;;;;;;; Airgap Tokens ;;;;;;;;;;;;;;;;;;;;@@ -245,33 +194,16 @@(when (> (t2/count :model/User :is_active true, :type :personal) max-users)(throw (Exception. (trs "You have reached the maximum number of users ({0}) for your plan. Please upgrade to add more users." max-users))))))-(mu/defn- fetch-token-status* :- TokenStatus- "Fetch info about the validity of `token` from the MetaStore."+(mu/defn- decode-token* :- TokenStatus+ "Decode a token. If you get a positive response about the token, even if it is not valid, return that. Errors will+ be caught further up with appropriate fall backs, retry strategies, and grace periods for features."[token :- TokenStr];; NB that we fetch any settings from this thread, not inside on of the futures in the inner fetch calls. We;; will have taken a lock to call through to here, and could create a deadlock with the future's thread. See;; https://github.com/metabase/metabase/pull/38029/(cond (mr/validate [:re RemoteCheckedToken] token)- ;; attempt to query the metastore API about the status of this token. If the request doesn't complete in a- ;; reasonable amount of time throw a timeout exception(let [site-uuid (premium-features.settings/site-uuid-for-premium-features-token-checks)]- (try (fetch-token-and-parse-body token token-check-url site-uuid)- (catch Exception e1- (log/errorf e1 "Error fetching token status from %s:" token-check-url)- ;; Try the fallback URL, which was the default URL prior to 45.2- (try (fetch-token-and-parse-body token store-url site-uuid)- ;; if there was an error fetching the token from both the normal and fallback URLs, log the- ;; first error and return a generic message about the token being invalid. This message- ;; will get displayed in the Settings page in the admin panel so we do not want something- ;; complicated- (catch Exception e2- (log/errorf e2 "Error fetching token status from %s:" store-url)- (let [body (u/ignore-exceptions (some-> (ex-data e1) :body json/decode+kw))]- (or- body- {:valid false- :status (tru "Unable to validate token")- :error-details (.getMessage e1)})))))))+ (fetch-token-and-parse-body token token-check-url site-uuid))(mr/validate [:re AirgapToken] token)(do@@ -287,40 +219,190 @@(def ^:dynamic *token-check-happening* "Var to prevent recursive calls to `fetch-token-status`" false)-(let [lock (Object.)]- (defn- fetch-token-status- "Locked version of `fetch-token-status*` allowing one request at a time."- [token]- (when *token-check-happening*- (throw (ex-info "Token check is being called recursively, there is a good chance some `defenterprise` is causing this"- {:pass-thru true})))- (locking lock- (binding [*token-check-happening* true]- (fetch-token-status* token)))))--(declare token-valid-now?)--(mu/defn- valid-token->features :- [:set ms/NonBlankString]- [token :- TokenStr]- (assert ((requiring-resolve 'metabase.app-db.core/db-is-set-up?)) "Metabase DB is not yet set up")- (let [{:keys [valid status features error-details] :as token-status} (fetch-token-status token)]- ;; if token isn't valid throw an Exception with the `:status` message- (when-not valid- (throw (ex-info status- {:status-code 400,- :error-details error-details})))- (when (and (mr/validate [:re AirgapToken] token)- (not (token-valid-now? token-status)))- (throw (ex-info status- {:status-code 400- :error-details (tru "Airgapped token is no longer valid. Please contact Metabase support.")})))- ;; otherwise return the features this token supports- (set features)))+(p/defprotocol+ GracePeriod+ "A protocol for providing a grace period for token features in the event they are not fetchable for a little while."+ (save! [_ token features] "Save the features for a particular token.")+ (retrieve [_ token] "Attempt to retrieve features associated with a token. This is best effort, perhaps never set,+ perhaps has timed out. Possible this is nil."))++(defn guava-cache-grace-period+ "Create a grace period of n units. This is just an expiring map using a guava cache. Note this is not sensitive to read times but to write times.+++ (let [grace (guava-cache-grace-period 20 TimeUnit/MILLISECONDS)]+ (save! grace \"token\" #{\"features\"})+ (println \"found tokens?: \"+ (if (= (retrieve grace \"token\") #{\"features\"})+ \"π’\"+ \"π΄\"))+ (Thread/sleep 40)+ (println \"expecting not to find tokens?: \"+ (if (retrieve grace \"token\")+ \"π΄\"+ \"π’\")))+ found tokens?: π’+ expecting not to find tokens?: π’+ nil"+ [^long n ^TimeUnit units]+ (let [guava-cache (.. (CacheBuilder/newBuilder)+ (expireAfterWrite n units)+ (removalListener (fn [^RemovalNotification rn]+ (let [cause (.getCause rn)]+ (when (= RemovalCause/EXPIRED cause)+ (log/warnf "Removing token: %s from grace period cache"+ (u.str/mask (.getKey rn)))))))+ (build))]+ (reify GracePeriod+ (save! [_ token features] (.put guava-cache token features))+ (retrieve [_ token]+ (when token+ (let [value (.get guava-cache token (constantly ::not-present))]+ (when-not (identical? value ::not-present)+ value)))))))++(p/defprotocol+ TokenChecker+ "Protocol for checking tokens with cache management."+ (-check-token [this token]+ "Check a token and return TokenStatus map. May throw exceptions on failure.")+ (-clear-cache! [this]+ "Clear any caches in this checker and any wrapped checkers.+ Returns nil. Implementations should delegate to wrapped checkers."))++(def store-and-airgap-token-checker+ "Creates a basic token checker that handles HTTP requests and airgap tokens.+ No caching, no retries, no grace period - just the core logic."+ (reify TokenChecker+ (-check-token [_ token]+ (decode-token* token))+ (-clear-cache! [_]+ ;; No cache to clear at this level+ nil)))++(defn circuit-breaker-token-checker+ "Wraps a token checker with circuit breaker and timeout logic."+ [token-checker {:keys [circuit-breaker timeout-ms lock]+ :or {lock (Object.)}}]+ (let [breaker (dh.cb/circuit-breaker circuit-breaker)]+ (reify TokenChecker+ (-check-token [_ token]+ (when *token-check-happening*+ (throw (ex-info "Token check is being called recursively, there is a good chance some `defenterprise` is causing this"+ {:pass-thru true})))+ (locking lock+ (binding [*token-check-happening* true]+ (try (dh/with-circuit-breaker breaker+ (dh/with-timeout {:timeout-ms timeout-ms+ :interrupt? true}+ (-check-token token-checker token)))+ (catch dev.failsafe.CircuitBreakerOpenException _e+ (throw (ex-info (tru "Token validation is currently unavailable.")+ {:cause :circuit-breaker})))+ ;; other exceptions are wrapped by Diehard in a FailsafeException. Unwrap them before+ ;; rethrowing.+ (catch dev.failsafe.FailsafeException e+ (throw (.getCause e)))))))+ (-clear-cache! [_]+ ;; No cache at this level, but delegate to wrapped checker+ (-clear-cache! token-checker)))))++(defn cached-token-checker+ "Wraps a token checker with TTL-based memoization."+ [token-checker {:keys [ttl-ms]}]+ (let [cached-check (memoize/ttl+ (fn [token] (-check-token token-checker token))+ :ttl/threshold ttl-ms)]+ (reify TokenChecker+ (-check-token [_ token]+ (cached-check token))+ (-clear-cache! [_]+ ;; Clear THIS layer's cache+ (memoize/memo-clear! cached-check)+ ;; AND delegate to wrapped checker+ (-clear-cache! token-checker)))))++(defn grace-period-token-checker+ "Wraps a token checker with grace period fallback logic."+ [token-checker {:keys [grace-period]}]+ (let [periodic-logger (memoize/ttl+ (fn [_token] (log/info "Using token from grace period"))+ :ttl/threshold (u/hours->ms 4))]+ (reify TokenChecker+ (-check-token [_ token]+ (try (let [response (-check-token token-checker token)]+ (save! grace-period token response)+ response)+ (catch Exception e+ (or (when-let [grace (retrieve grace-period token)]+ (periodic-logger token)+ grace)+ (throw e)))))+ (-clear-cache! [_]+ ;; The grace period itself doesn't need clearing (it expires naturally)+ ;; but we should clear the periodic logger cache+ (memoize/memo-clear! periodic-logger)+ ;; AND delegate to wrapped checker+ (-clear-cache! token-checker)))))++(defn- error-catching-token-checker+ [token-checker]+ (reify TokenChecker+ (-check-token [_ token]+ (try (-check-token token-checker token)+ (catch Exception e+ (u/ignore-exceptions (some-> (ex-data e) :body json/decode+kw))+ {:valid false+ :status (tru "Unable to validate token")+ :error-details (.getMessage e)})))+ (-clear-cache! [_] (-clear-cache! token-checker))))++(def ^:dynamic *customize-checker*+ "Dynamic variable allowing for customized token checkers. In the app, we want all of these in place. Only in tests+ should we construct ones without circuit breakers "+ false)-(defn -token-status- "Getter for the [[metabase.premium-features.settings/token-status]] setting."+(defn make-checker+ "Make a token checker. Takes a base [[TokenChecker]] token checker and then arguments for the middleware-style+ wrapping [[TokenChecker]] arguments. "+ [{:keys [base circuit-breaker timeout-ms ttl-ms grace-period]+ :or {base store-and-airgap-token-checker}}]+ (when-not *customize-checker*+ (assert (and base circuit-breaker timeout-ms ttl-ms grace-period)+ "Must provide all arguments for token checker"))+ (cond-> base+ ;; don't do it too much+ circuit-breaker+ (circuit-breaker-token-checker {:circuit-breaker circuit-breaker+ :timeout-ms timeout-ms})+ ;; hold onto results for a while+ ttl-ms+ (cached-token-checker {:ttl-ms ttl-ms})+ ;; in case of errors, if we have a recent response use it+ grace-period+ (grace-period-token-checker {:grace-period grace-period})+ :always+ (error-catching-token-checker)))++(def token-checker+ "The token checker. Combines http/airgapping vaildation, circuit breaking, grace periods, caching, and error+ handling."+ (make-checker {:base store-and-airgap-token-checker+ :circuit-breaker {:failure-threshold-ratio-in-period [10 10 (u/seconds->ms 10)]+ :delay-ms (u/seconds->ms 30)+ :success-threshold 1}+ :timeout-ms (u/seconds->ms 10)+ :ttl-ms (u/hours->ms 12)+ :grace-period (guava-cache-grace-period 36 TimeUnit/HOURS)}))++(defn clear-cache!+ "Clear the token cache so that [[fetch-token-and-parse-body]] will return the latest data."[]- (some-> (premium-features.settings/premium-embedding-token) (fetch-token-status)))+ (-clear-cache! token-checker))++(defn check-token+ "Public entrypoint to the token checking."+ ([token] (check-token token-checker token))+ ([checker token]+ (-check-token checker token)))(defn -set-premium-embedding-token!"Setter for the [[metabase.premium-features.settings/token-status]] setting."@@ -334,7 +416,9 @@(mr/validate [:re AirgapToken] new-value))(throw (ex-info (tru "Token format is invalid."){:status-code 400, :error-details "Token should be 64 hexadecimal characters."})))- (valid-token->features new-value)+ (let [decoded (check-token new-value)]+ (when-not (:valid decoded)+ (throw (ex-info "Invalid token" {:token (u.str/mask new-value)}))))(log/info "Token is valid."))(setting/set-value-of-type! :string :premium-embedding-token new-value)(catch Throwable e@@ -360,7 +444,9 @@"Get the features associated with the system's premium features token."[](try- (or (some-> (premium-features.settings/premium-embedding-token) valid-token->features)+ (or (some-> (premium-features.settings/premium-embedding-token)+ (check-token)+ :features set)#{})(catch Throwable e(when (:pass-thru (ex-data e))@@ -368,19 +454,25 @@(cached-logger (premium-features.settings/premium-embedding-token) e)#{}))))+(defn -token-status+ "Getter for the [[metabase.premium-features.settings/token-status]] setting."+ []+ (some-> (premium-features.settings/premium-embedding-token)+ (check-token)))+(mu/defn plan-alias :- [:maybe :string]"Returns a string representing the instance's current plan, if included in the last token status request."[](some-> (premium-features.settings/premium-embedding-token)- fetch-token-status+ (check-token):plan-alias))(mu/defn quotas :- [:maybe [:sequential [:map]]]"Returns a vector of maps for each quota of the subscription."[]- (memoize/memo-clear! fetch-token-and-parse-body*)+ (clear-cache!)(some-> (premium-features.settings/premium-embedding-token)- fetch-token-status+ (check-token):quotas))(defn has-any-features?@@ -430,9 +522,3 @@metabase-enterprise.premium-features.airgap[_]{})--(defenterprise token-valid-now?- "In OSS, this returns false."- metabase-enterprise.premium-features.airgap- [_]- false)diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.cljindex 96bc413fad0..ff6c2d20709 100644--- a/test/metabase/premium_features/token_check_test.clj+++ b/test/metabase/premium_features/token_check_test.clj@@ -1,93 +1,33 @@(ns metabase.premium-features.token-check-test(:require[clj-http.client :as http]- [clj-http.fake :as http-fake][clojure.test :refer :all]- [diehard.circuit-breaker :as dh.cb][mb.hawk.parallel][metabase.app-db.connection :as mdb.connection]- [metabase.config.core :as config][metabase.premium-features.core :as premium-features]- [metabase.premium-features.settings :as premium-features.settings][metabase.premium-features.test-util :as tu][metabase.premium-features.token-check :as token-check][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [metabase.util.json :as json][metabase.util.malli.registry :as mr]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.util.concurrent TimeUnit)))(set! *warn-on-reflection* true)-(defn- open-circuit-breaker! [cb]- (.open ^dev.failsafe.CircuitBreaker cb))--(defmacro with-open-circuit-breaker! [& body]- `(binding [token-check/*store-circuit-breaker* (dh.cb/circuit-breaker- @#'token-check/store-circuit-breaker-config)]- (open-circuit-breaker! token-check/*store-circuit-breaker*)- (do ~@body)))--(defn reset-circuit-breaker-fixture [f]- (binding [token-check/*store-circuit-breaker* (dh.cb/circuit-breaker- @#'token-check/store-circuit-breaker-config)]- (f)))-(use-fixtures :once (fixtures/initialize :db))-(use-fixtures :each reset-circuit-breaker-fixture)-(defn- token-status-response- [token token-check-response]- (http-fake/with-fake-routes-in-isolation- {{:address (#'token-check/token-status-url token @#'token-check/token-check-url)- :query-params (merge (#'token-check/stats-for-token-request)- {:site-uuid (premium-features.settings/site-uuid-for-premium-features-token-checks)- :mb-version (:tag config/mb-version-info)})}- (constantly token-check-response)}- (#'token-check/fetch-token-status* token)))--(def ^:private token-response-fixture- (json/encode {:valid true- :status "fake"- :features ["test" "fixture"]- :trial false}))--(deftest ^:parallel fetch-token-status-test+(deftest log-tests(let [token (tu/random-token)print-token (apply str (concat (take 4 token) "..." (take-last 4 token)))](testing "Do not log the token (#18249)"(mt/with-log-messages-for-level [messages :info]- (#'token-check/fetch-token-status* token)+ (token-check/check-token token)(let [logs (mapv :message (messages))](is (every? (complement #(re-find (re-pattern token) %)) logs))(is (= 1 (count (filter #(re-find (re-pattern print-token) %) logs)))))))))-(deftest ^:parallel fetch-token-status-test-2- (testing "With the backend unavailable"- (let [result (token-status-response (tu/random-token) {:status 500})]- (is (false? (:valid result))))))--(deftest ^:parallel fetch-token-status-test-3- (testing "On other errors"- (binding [http/request (fn [& _]- ;; note originally the code caught clojure.lang.ExceptionInfo so don't- ;; throw an ex-info here- (throw (Exception. "network issues")))]- (is (= {:valid false- :status "Unable to validate token"- :error-details "network issues"}- (#'token-check/fetch-token-status (apply str (repeat 64 "b"))))))))--(deftest fetch-token-caches-successful-responses- (testing "For successful responses, the result is cached"- (let [call-count (atom 0)- token (tu/random-token)]- (binding [http/request (fn [& _]- (swap! call-count inc)- {:status 200 :body "{\"valid\": true, \"status\": \"fake\"}"})]- (dotimes [_ 10] (#'token-check/fetch-token-status token))- (is (= 1 @call-count))))))-(deftest fetch-token-caches-invalid-responses(testing "For 4XX responses, the result is cached"(let [call-count (atom 0)@@ -95,91 +35,183 @@(binding [http/request (fn [& _](swap! call-count inc){:status 400 :body "{\"valid\": false, \"status\": \"fake\"}"})]- (dotimes [_ 10] (#'token-check/fetch-token-status token))+ (token-check/-clear-cache! token-check/token-checker)+ (dotimes [_ 10] (token-check/check-token token))(is (= 1 @call-count))))))(deftest fetch-token-does-not-cache-exceptions- (testing "For timeouts, 5XX errors, etc. we don't cache the result"- (let [call-count (atom 0)- token (tu/random-token)]- (binding [http/request (fn [& _]- (swap! call-count inc)- (throw (ex-info "oh, fiddlesticks" {})))]- (dotimes [_ 5] (#'token-check/fetch-token-status token))- ;; Note that we have a fallback URL that gets hit in this case (see- ;; https://github.com/metabase/metabase/issues/27036) and 2x5=10- (is (= 10 @call-count))))))--(deftest fetch-token-does-not-cache-5XX-responses(let [call-count (atom 0)- token (tu/random-token)]- (binding [http/request (fn [& _]- (swap! call-count inc)- {:status 500})]- (dotimes [_ 10] (#'token-check/fetch-token-status token))- ;; Same as above, we have a fallback URL that gets hit in this case (see- ;; https://github.com/metabase/metabase/issues/27036) and 2x10=20- (is (= 10 @call-count)))))--(deftest fetch-token-is-circuit-broken- (let [call-count (atom 0)]- (with-open-circuit-breaker!- (binding [http/request (fn [& _] (swap! call-count inc))]- (is (= {:valid false- :status "Unable to validate token"- :error-details "Token validation is currently unavailable."}- (#'token-check/fetch-token-status (tu/random-token))))- (is (= 0 @call-count))))))--(deftest ^:parallel fetch-token-status-test-4- (testing "With a valid token"- (let [result (token-status-response (tu/random-token) {:status 200- :body token-response-fixture})]- (is (:valid result))- (is (contains? (set (:features result)) "test")))))+ token (tu/random-token)+ response (atom :error)+ ;; i've removed the circuit breaker from the stack. that sometimes shuts down the tests in a way we don't want+ ;; to exercise here+ checker (binding [token-check/*customize-checker* true]+ (token-check/make-checker {:ttl-ms 500+ :grace-period (token-check/guava-cache-grace-period 36 TimeUnit/HOURS)}))]+ (with-redefs [token-check/http-fetch (fn [& _]+ (swap! call-count inc)+ (case @response+ :error (throw (ex-info "kaboom" {:ka :boom}))+ :500 {:status 500}))]+ (testing "For timeouts, 5XX errors, etc. we don't cache the result"+ (dotimes [_ 5] (token-check/check-token checker token))+ (is (= 5 @call-count)))+ (testing "Does not cache 500 responses"+ (reset! call-count 0)+ (reset! response :500)+ (dotimes [_ 5] (token-check/check-token checker token))+ (is (= 5 @call-count))))))(deftest not-found-test(mt/with-log-level :fatal- ;; `partial=` here in case the Cloud API starts including extra keys... this is a "dangerous" test since changes- ;; upstream in Cloud could break this. We probably want to catch that stuff anyway tho in tests rather than waiting- ;; for bug reports to come in- (is (partial= {:valid false, :status "Token does not exist."}- (#'token-check/fetch-token-status* (tu/random-token))))))+ (is (=? {:valid false, :status "Token does not exist."}+ (token-check/check-token (tu/random-token))))))(deftest fetch-token-does-not-call-db-when-cached(testing "No DB calls are made when checking token status if the status is cached"(let [token (tu/random-token)- _ (#'token-check/fetch-token-status token)+ _ (token-check/check-token token);; Sigh. This is really quite horrific. But we need some wiggle room here: any endpoint that gets some setting;; inside it is going to check to see whether it's time for an update check. If it is, it'll hit the DB to see;; when settings were last updated, and the count will be incremented. Therefore, let's do this a few times...call-counts (repeatedly 3 (fn [](t2/with-call-count [call-count]- (#'token-check/fetch-token-status token)+ (token-check/check-token token)(call-count))))];; ... and then make sure that *some* of the times, we didn't hit the DB again.(is (some zero? call-counts)))))+(deftest token-checker-test+ (let [token (tu/random-token)+ behavior (atom :success)+ call-count (atom 0)+ good-response {:valid true+ :status :fake+ :features [:feature1 :feature2]}+ no-features {:valid false :status "Unable to validate token" :error-details "network issues!"}+ token-response (fn [_token]+ (swap! call-count inc)+ (case @behavior+ :success good-response+ :timeout (Thread/sleep 200)+ :error (throw (ex-info "network issues!" {:ka :boom}))))+ checker (token-check/make-checker+ {:base (reify token-check/TokenChecker+ (-check-token [_ token]+ (token-response token))+ (-clear-cache! [_]))+ :circuit-breaker {:failure-threshold-ratio-in-period [4 4 1000]+ :delay-ms 50+ :success-threshold 1}+ :timeout-ms 100+ :ttl-ms 200+ :grace-period (token-check/guava-cache-grace-period 1000 TimeUnit/MILLISECONDS)})]+ (testing "when no information is present, hits the underlying"+ (is (= good-response (token-check/-check-token checker token)))+ (is (= 1 @call-count)))+ (testing "hitting it again in the same timeperiod does not hit the underlying checker"+ (is (= good-response (token-check/-check-token checker token)))+ (is (= 1 @call-count)))+ (testing "While it errors, we have circuit breaking"+ (reset! behavior :error)+ (Thread/sleep 200)+ (dotimes [_ 1000] (token-check/-check-token checker token))+ (is (< @call-count 50))+ (testing "But we always get a good response from the grace period"+ (is (= good-response (token-check/-check-token checker token)))))+ (testing "But eventually grace period elapses"+ (Thread/sleep 1000)+ (is (= no-features (token-check/-check-token checker token))))+ (testing "When connectivity is restored we get successful token checks"+ (reset! behavior :success)+ (Thread/sleep 100) ;; wait for circuit breaker to open back up+ (is (= good-response (token-check/-check-token checker token))))))++(deftest grace-period-test+ (testing "implementation check"+ (let [token (tu/random-token)+ behavior (atom :success)+ success-token {:valid true :status :fake :features [:feature1 :feature2]}+ restored-token {:valid true :status :fake :features [:new-feature]}+ invalid {:valid false, :status "Unable to validate token", :error-details nil}+ underlying (reify token-check/TokenChecker+ (-check-token [_ t]+ (when (= t token)+ (case @behavior+ :success success-token+ :error (throw (ex-info "network troubles!" {:ka :boom}))+ :network-restored restored-token)))+ (-clear-cache! [_]))+ grace (binding [token-check/*customize-checker* true]+ (token-check/make-checker+ {:base underlying+ :grace-period (token-check/guava-cache-grace-period 20 TimeUnit/MILLISECONDS)}))]+ (is (= success-token (token-check/check-token grace token)))+ (is (= invalid (token-check/check-token grace (tu/random-token))))+ (testing "During errors"+ (reset! behavior :error)+ (is (= success-token (token-check/check-token grace token)))+ (is (= invalid (token-check/check-token grace (tu/random-token))))+ (Thread/sleep 40) ;; our simple one here has a grace period of 20 milliseconds+ (testing "but it expires"+ (is (= {:valid false :status "Unable to validate token" :error-details "network troubles!"}+ (token-check/check-token grace token)))))+ (testing "When good"+ (reset! behavior :success)+ (is (= success-token (token-check/check-token grace token))))+ (testing "We can enter the grace period"+ (reset! behavior :error)+ (is (= success-token (token-check/check-token grace token))))+ (testing "And then we immediately get new token when network restores"+ (reset! behavior :network-restored)+ (is (= restored-token (token-check/check-token grace token)))))))++(deftest e2e+ (testing "token check hits the network"+ (let [token (tu/random-token)+ ;; todo: need to check that we use the grace period+ time-passes! token-check/clear-cache!+ call-count (atom 0)]+ (with-redefs [token-check/http-fetch (fn [& _]+ (swap! call-count inc)+ {:status 200+ :body "{\"valid\":true,\"status\":\"fake\",\"features\":[\"fake\",\"features\"]}"})]+ (= {:valid true, :status "fake", :features ["fake" "features"]}+ (token-check/check-token token))+ (is (= 1 @call-count)))+ (with-redefs [token-check/http-fetch (fn [& _]+ (swap! call-count inc)+ (throw (ex-info "network failure!" {})))]+ (testing "Doesn't hit the network inside of cache duration"+ (is (= {:valid true, :status "fake", :features ["fake" "features"]}+ (token-check/check-token token)))+ (is (= 1 @call-count)))+ (time-passes!) ;; we can clear the cache but don't have a great way to expire the grace period+ (testing "Hits the network periodically (12 hours)"+ (let [response (token-check/check-token token)]+ ;; called but we got network failure from redefs+ (is (= 2 @call-count))+ (testing "Grace period supplements errors"+ (is (= {:valid true, :status "fake", :features ["fake" "features"]}+ response)))))))))+(deftest token-status-setting-test(testing "If a `premium-embedding-token` has been set, the `token-status` setting should return the responsefrom the store.metabase.com endpoint for that token."- (mt/with-temporary-raw-setting-values [premium-embedding-token (tu/random-token)]- (is (= {:valid false, :status "Token does not exist."}- (premium-features/token-status)))))+ (is (= {:valid false, :status "Token does not exist."}+ (token-check/check-token (tu/random-token)))))(testing "If premium-embedding-token is nil, the token-status setting should also be nil."(mt/with-temporary-setting-values [premium-embedding-token nil](is (nil? (premium-features/token-status))))))(deftest active-users-count-setting-test- (mt/with-temp- [:model/User _ {:is_active false}]- (testing "returns the number of active users"- (is (= (t2/count :model/User :is_active true :type :personal)- (premium-features/active-users-count))))+ (testing "returns the number of active users"+ (is (= (t2/count :model/User :is_active true :type :personal)+ (premium-features/active-users-count))))- (testing "Default to 0 if db is not setup yet"- (binding [mdb.connection/*application-db* {:status (atom nil)}]- (is (zero? (premium-features/active-users-count)))))))+ (testing "Default to 0 if db is not setup yet"+ (binding [mdb.connection/*application-db* {:status (atom nil)}]+ (is (zero? (premium-features/active-users-count))))))(deftest RemoteCheckedToken-regexp(testing "valid tokens"diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.cljindex c8c5819ceb4..9c3e7193672 100644--- a/test/metabase/test/util.clj+++ b/test/metabase/test/util.clj@@ -600,7 +600,7 @@The token-value binding will contain the random token that was set."[[token-value] & body]`(let [~token-value (premium-features.test-util/random-token)]- (with-redefs [metabase.premium-features.token-check/fetch-token-status+ (with-redefs [metabase.premium-features.token-check/check-token(constantly {:valid true:status "fake":features ["test" "fixture"]