The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Move metering events off the token check (#67584)
Nicola Mometto4e3d8f0caf4c
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex e9a08b6bb88..292ee757207 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -953,6 +953,7 @@configinternal-statssettings+ taskutil}}product-feedbackdiff --git a/src/metabase/premium_features/init.clj b/src/metabase/premium_features/init.cljindex d6845e05413..3708aa217b7 100644--- a/src/metabase/premium_features/init.clj+++ b/src/metabase/premium_features/init.clj@@ -1,4 +1,5 @@(ns metabase.premium-features.init(:require[metabase.premium-features.core]- [metabase.premium-features.settings]))+ [metabase.premium-features.settings]+ [metabase.premium-features.task.send-metering]))diff --git a/src/metabase/premium_features/settings.clj b/src/metabase/premium_features/settings.cljindex 6b8af935b2e..9a455a66b11 100644--- a/src/metabase/premium_features/settings.clj+++ b/src/metabase/premium_features/settings.clj@@ -379,3 +379,11 @@:setter :none:getter -token-features:doc false)++(defsetting send-metering-interval-ms+ "Interval in milliseconds between metering event sends."+ :type :integer+ :default nil+ :visibility :internal+ :export? false+ :doc false)diff --git a/src/metabase/premium_features/task/send_metering.clj b/src/metabase/premium_features/task/send_metering.cljnew file mode 100644index 00000000000..dc86a68136b--- /dev/null+++ b/src/metabase/premium_features/task/send_metering.clj@@ -0,0 +1,48 @@+(ns metabase.premium-features.task.send-metering+ "Scheduled task that sends metering events for billing purposes."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.simple :as simple]+ [clojurewerkz.quartzite.triggers :as triggers]+ [metabase.premium-features.settings :as settings]+ [metabase.premium-features.token-check :as token-check]+ [metabase.task.core :as task]+ [metabase.util :as u]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(def ^:private min-interval-ms (u/minutes->ms 1))+(def ^:private max-interval-ms (u/hours->ms 12))+(def ^:private default-interval-ms (u/minutes->ms 15))++(defn- metering-interval-ms+ "Returns the metering interval in milliseconds, clamped between 1 minute and 12 hours.+ Defaults to 15 minutes if not set."+ []+ (if-let [v (settings/send-metering-interval-ms)]+ (max min-interval-ms (min max-interval-ms v))+ default-interval-ms))++(task/defjob ^{:doc "Send metering events for billing purposes"}+ SendMeteringEvents+ [_]+ (log/debug "Running metering events task")+ (token-check/send-metering-events!))++(def ^:private job-key "metabase.task.send-metering.job")+(def ^:private trigger-key "metabase.task.send-metering.trigger")++(defmethod task/init! ::SendMeteringEvents+ [_]+ (let [job (jobs/build+ (jobs/of-type SendMeteringEvents)+ (jobs/with-identity (jobs/key job-key)))+ trigger (triggers/build+ (triggers/with-identity (triggers/key trigger-key))+ (triggers/start-now)+ (triggers/with-schedule+ (simple/schedule+ (simple/with-interval-in-milliseconds (metering-interval-ms))+ (simple/repeat-forever))))]+ (task/schedule-task! job trigger)))diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex cb5f1b07cab..67b57a18eb5 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -35,7 +35,7 @@(set! *warn-on-reflection* true)-(def ^:private RemoteCheckedToken+(def RemoteCheckedToken"Schema for a valid premium token. Must be 64 lower-case hex characters."#"^(mb_dev_[0-9a-f]{57}|[0-9a-f]{64})$")@@ -48,7 +48,7 @@[:re RemoteCheckedToken][:re AirgapToken]])-(def ^:private ^String token-check-url+(def ^String token-check-url"Base URL to use for token checks. Hardcoded by default but for development purposes you can use a local server.Specify the env var `METASTORE_DEV_SERVER_URL`. If no server is defined, it uses the staging token check url."(let [dev-server-url (some-> (env :metastore-dev-server-url) (str/replace #"/$" ""))@@ -115,7 +115,8 @@t/local-datestr)})-(defn- stats-for-token-request+(defn metering-stats+ "Collect metering statistics for billing purposes. Used by both token check and metering task. "[];; NOTE: beware, if you use `defenterprise` here which uses any other `:feature` other than `:none`, it will;; recursively trigger token check and will die@@ -158,10 +159,9 @@(defn- http-fetch[base-url token site-uuid](some-> (token-status-url token base-url)- (http/get {:query-params (merge (stats-for-token-request)- {:site-uuid site-uuid- :mb-version (:tag config/mb-version-info)})- :throw-exceptions false+ (http/get {:query-params {:site-uuid site-uuid+ :mb-version (:tag config/mb-version-info)}+ :throw-exceptions false;; socket is data transfer, connection is handshake and create connection timeout:socket-timeout 5000 ;; in milliseconds:connection-timeout 2000 ;; in milliseconds@@ -182,6 +182,29 @@:else (throw (ex-info "An unknown error occurred when validating token." {:status status:body body})))))+(defn- metering-url+ [token base-url]+ (format "%s/api/%s/v2/metering" base-url token))++(defn send-metering-events!+ "Send metering events for billing purposes"+ []+ (when-let [token (premium-features.settings/premium-embedding-token)]+ (when (mr/validate [:re RemoteCheckedToken] token)+ (let [site-uuid (premium-features.settings/site-uuid-for-premium-features-token-checks)+ stats (-> (metering-stats)+ ;; for backwards compatibility, we send values as strings+ (update-vals str))]+ (try+ (http/post (metering-url token token-check-url)+ {:body (json/encode (merge stats+ {:site-uuid site-uuid+ :mb-version (:tag config/mb-version-info)}))+ :content-type :json+ :throw-exceptions false})+ (catch Throwable e+ (log/error e "Error sending metering events")))))))+;;;;;;;;;;;;;;;;;;;; Airgap Tokens ;;;;;;;;;;;;;;;;;;;;(declare decode-airgap-token)diff --git a/test/metabase/premium_features/token_check_test.clj b/test/metabase/premium_features/token_check_test.cljindex 932863dc4ec..fd54d7a3b0a 100644--- a/test/metabase/premium_features/token_check_test.clj+++ b/test/metabase/premium_features/token_check_test.clj@@ -1,6 +1,7 @@(ns metabase.premium-features.token-check-test(:require[clj-http.client :as http]+ [clj-http.core :as http.core][clojure.test :refer :all][mb.hawk.parallel][metabase.app-db.connection :as mdb.connection]@@ -10,6 +11,7 @@[metabase.premium-features.token-check :as token-check][metabase.test :as mt][metabase.test.fixtures :as fixtures]+ [metabase.util.json :as json][metabase.util.malli.registry :as mr][toucan2.core :as t2])(:import@@ -301,3 +303,72 @@(is (thrown-with-msg? Exception#"Adding another user would exceed the maximum"(token-check/assert-airgap-allows-user-creation!))))))++(deftest send-metering-events-test+ (testing "send-metering-events! makes a POST request with correct data"+ (let [request-data (atom nil)]+ (mt/with-random-premium-token! [_token]+ (with-redefs [http/post (fn [url opts]+ (reset! request-data {:url url :opts opts})+ {:status 200 :body "{}"})]+ (token-check/send-metering-events!)+ (is (some? @request-data) "POST request should have been made")+ (when @request-data+ (is (re-find #"/v2/metering$" (:url @request-data))+ "URL should end with /v2/metering")+ (is (= :json (get-in @request-data [:opts :content-type]))+ "Content-Type should be JSON")+ (is (false? (get-in @request-data [:opts :throw-exceptions]))+ "throw-exceptions should be false")+ (let [body (json/decode (get-in @request-data [:opts :body]))]+ (is (contains? body "site-uuid")+ "Request body should include site-uuid")+ (is (contains? body "mb-version")+ "Request body should include mb-version")+ (is (contains? body "users")+ "Request body should include users count"))))))))++(deftest send-metering-events-error-handling-test+ (testing "send-metering-events! handles errors gracefully"+ (mt/with-random-premium-token! [_token]+ (with-redefs [http.core/request (fn [& _] (throw (Exception. "Network error")))]+ ;; Should not throw, just log the error+ (is (nil? (token-check/send-metering-events!)))))))++(deftest send-metering-events-no-token-test+ (testing "send-metering-events! does nothing when no token is set"+ (let [request-made (atom false)]+ (mt/with-temporary-setting-values [premium-embedding-token nil]+ (with-redefs [http/post (fn [_url _opts]+ (reset! request-made true)+ {:status 200 :body "{}"})]+ (token-check/send-metering-events!)+ (is (false? @request-made) "No request should be made without a token"))))))++(deftest send-metering-events-airgap-token-test+ (testing "send-metering-events! does nothing for airgap tokens"+ (let [;; This is a fake airgap token format (starts with "airgap_")+ airgap-token "airgap_eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ90faketoken"+ request-made (atom false)]+ (with-redefs [token-check/check-token+ (constantly {:valid true+ :status "fake"+ :features ["test" "fixture"]+ :trial false})]+ (mt/with-temporary-raw-setting-values [premium-embedding-token airgap-token]+ (with-redefs [http/post (fn [_url _opts]+ (reset! request-made true)+ {:status 200 :body "{}"})]+ (token-check/send-metering-events!)+ (is (false? @request-made) "No request should be made for airgap tokens")))))))++(deftest metering-stats-test+ (testing "metering-stats returns expected keys"+ (let [stats (token-check/metering-stats)]+ (is (map? stats))+ (is (contains? stats :users))+ (is (contains? stats :external-users))+ (is (contains? stats :internal-users))+ (is (contains? stats :domains))+ (is (contains? stats :embedding-dashboard-count))+ (is (contains? stats :embedding-question-count)))))