The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix: fix transform run metering aggregation (#72632)
Filipe Silva9571b2b4c3dd
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.cljindex 92c1bc4cf62..63d9326da8c 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj@@ -10,27 +10,50 @@[metabase.premium-features.core :as premium-features :refer [defenterprise]][toucan2.core :as t2]))+(defenterprise transform-metered-as+ "Return the meter bucket a new transform run of the given source-type counts toward,+ based on the instance's current premium features. Returns nil when the run is not metered."+ :feature :none+ [source-type]+ (let [hosted? (premium-features/has-feature? :hosting)+ basic? (premium-features/has-feature? :transforms-basic)+ writable? (premium-features/has-feature? :writable-connection)+ python? (premium-features/has-feature? :transforms-python)]+ (case (keyword source-type)+ (:native :mbql) (cond+ ;; These transforms are metered differently based on plan and addons status:+ ;; - the advanced transforms addon adds the writable-connection feature, which+ ;; augments what they can do and meters them as advanced+ (and basic? writable?) "transform-advanced"+ ;; - hosted instances with basic transforms get these metered as basic+ (and basic? hosted?) "transform-basic"+ ;; - self-hosted customers without the advanced add-on aren't metered for these at all+ :else nil)+ :python (when python? "transform-advanced")+ nil)))+(defenterprise transform-stats"Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.- Rolling stats are included under :transform-rolling (up to date totals for today)"+ Rolling stats are included under :transform-rolling (up to date totals for today).++ Runs are bucketed by the :metered_as column set at `start-run!` time.":feature :none[]- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))- yesterday-utc (t/minus today-utc (t/days 1))- count-all-runs (fn [date]- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]- :from [[:transform_run :r]]- :where [:and- [:= :r.status "succeeded"]- [:= [:cast :r.end_time :date] [:cast date :date]]]}))- 0))- advanced? (and (premium-features/has-feature? :transforms-basic)- (premium-features/has-feature? :transforms-python))- yesterday-runs (count-all-runs yesterday-utc)- today-runs (count-all-runs today-utc)]- {:transform-basic-runs (if advanced? 0 yesterday-runs)- :transform-advanced-runs (if advanced? yesterday-runs 0)- :transform-usage-date (str (t/local-date yesterday-utc))- :transform-rolling-basic-runs (if advanced? 0 today-runs)- :transform-rolling-advanced-runs (if advanced? today-runs 0)- :transform-rolling-usage-date (str (t/local-date today-utc))}))+ (let [today-utc (t/offset-date-time (t/zone-offset "+00"))+ yesterday-utc (t/minus today-utc (t/days 1))+ counts-for (fn [date]+ (into {} (map (juxt :metered_as :cnt))+ (t2/query {:select [:r.metered_as [[:count :r.id] :cnt]]+ :from [[:transform_run :r]]+ :where [:and+ [:= :r.status "succeeded"]+ [:= [:cast :r.end_time :date] [:cast date :date]]]+ :group-by [:r.metered_as]})))+ yesterday (counts-for yesterday-utc)+ today (counts-for today-utc)]+ {:transform-basic-runs (get yesterday "transform-basic" 0)+ :transform-advanced-runs (get yesterday "transform-advanced" 0)+ :transform-usage-date (str (t/local-date yesterday-utc))+ :transform-rolling-basic-runs (get today "transform-basic" 0)+ :transform-rolling-advanced-runs (get today "transform-advanced" 0)+ :transform-rolling-usage-date (str (t/local-date today-utc))}))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/core_test.cljindex db1d607fb73..86f5e0b0b3f 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj@@ -2,6 +2,8 @@(:require[clojure.test :refer :all][java-time.api :as t]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata][metabase.premium-features.core :as premium-features][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -12,52 +14,90 @@(use-fixtures :once (fixtures/initialize :db))-(deftest transform-stats-tier-based-metering-test- (testing "all runs are bucketed into basic or advanced based on the transforms-python feature flag"- (mt/with-temp [:model/Transform {native-id :id} {}- :model/Transform {python-id :id} {:source {:type "python"- :source-database (mt/id)}- :target {:type "table"- :name (str "test_python_" (random-uuid))- :database (mt/id)}}]- (let [stats-before (premium-features/transform-stats)- basic-before (:transform-basic-runs stats-before)- advanced-before (:transform-advanced-runs stats-before)- rolling-basic-before (:transform-rolling-basic-runs stats-before)- rolling-advanced-before (:transform-rolling-advanced-runs stats-before)- yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]-- (testing "without advanced add-on, all runs go to basic-runs"- ;; run a native transform- (let [{run-id :id} (transform-run/start-run! native-id {:run_method "manual"})]- (transform-run/succeed-started-run! run-id)-- (testing "completed run appears in rolling-basic-runs"- (let [stats (premium-features/transform-stats)]- (is (= (inc rolling-basic-before) (:transform-rolling-basic-runs stats)))- (is (= rolling-advanced-before (:transform-rolling-advanced-runs stats)))- (is (= basic-before (:transform-basic-runs stats)))))-- (testing "backdated run appears in basic-runs"- (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})- (let [stats (premium-features/transform-stats)]- (is (= rolling-basic-before (:transform-rolling-basic-runs stats)))- (is (= (inc basic-before) (:transform-basic-runs stats)))- (is (= advanced-before (:transform-advanced-runs stats)))))))-- (testing "with advanced add-on, all runs go to advanced-runs"- (mt/with-premium-features #{:transforms-basic :transforms-python}- ;; run a python transform- (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]- (transform-run/succeed-started-run! run-id)-- (testing "completed run appears in rolling-advanced-runs"- (let [stats (premium-features/transform-stats)]- (is (= 0 (:transform-rolling-basic-runs stats)))- (is (pos? (:transform-rolling-advanced-runs stats)))))-- (testing "backdated run appears in advanced-runs"- (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})- (let [stats (premium-features/transform-stats)]- (is (= 0 (:transform-basic-runs stats)))- (is (pos? (:transform-advanced-runs stats))))))))))))+(deftest transform-metered-as-test+ (mt/with-premium-features #{:transforms-basic :writable-connection}+ (is (= "transform-advanced"+ (premium-features/transform-metered-as :native)+ (premium-features/transform-metered-as :mbql))))++ (mt/with-premium-features #{:hosting :transforms-basic}+ (is (= "transform-basic"+ (premium-features/transform-metered-as :native)+ (premium-features/transform-metered-as :mbql))))++ (mt/with-premium-features #{:transforms-python}+ (is (= "transform-advanced"+ (premium-features/transform-metered-as :python))))++ (mt/with-premium-features #{}+ (is (= nil+ (premium-features/transform-metered-as :native)+ (premium-features/transform-metered-as :mbql))))++ (mt/with-premium-features #{:hosting :transforms-basic :transforms-python :writable-connection}+ (is (= nil+ (premium-features/transform-metered-as nil)+ (premium-features/transform-metered-as :something-else)))))++(deftest transform-stats-aggregation-test+ (testing "transform-stats buckets runs by the :metered_as captured at start-run! time"+ (let [frozen-today (t/offset-date-time 2037 6 15 12 0 0 0 (t/zone-offset "+00"))+ frozen-yesterday (t/minus frozen-today (t/days 1))+ mp (mt/metadata-provider)]+ (with-redefs [t/offset-date-time (constantly frozen-today)]+ (mt/with-model-cleanup [:model/TransformRun]+ (mt/with-temp [:model/Transform {mbql-id :id}+ {:source {:type "query"+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}+ :target {:type "table"+ :name (str "test_mbql_" (random-uuid))+ :database (mt/id)}}+ :model/Transform {python-id :id}+ {:source {:type "python"+ :source-database (mt/id)}+ :target {:type "table"+ :name (str "test_python_" (random-uuid))+ :database (mt/id)}}]+ (let [run! (fn [transform-id end-time]+ (let [{id :id} (transform-run/start-run! transform-id {:run_method "manual"})]+ (transform-run/succeed-started-run! id)+ (t2/update! :model/TransformRun :id id {:end_time end-time})))]++ (testing "with no premium features, mbql runs are not metered"+ (mt/with-premium-features #{}+ (run! mbql-id frozen-yesterday)+ (run! mbql-id frozen-today))+ (is (= {:transform-basic-runs 0+ :transform-advanced-runs 0+ :transform-usage-date "2037-06-14"+ :transform-rolling-basic-runs 0+ :transform-rolling-advanced-runs 0+ :transform-rolling-usage-date "2037-06-15"}+ (premium-features/transform-stats))))++ (testing "with :hosting :transforms-basic, mbql runs meter as basic"+ (mt/with-premium-features #{:hosting :transforms-basic}+ (run! mbql-id frozen-yesterday)+ (run! mbql-id frozen-today))+ (is (= {:transform-basic-runs 1+ :transform-advanced-runs 0+ :transform-usage-date "2037-06-14"+ :transform-rolling-basic-runs 1+ :transform-rolling-advanced-runs 0+ :transform-rolling-usage-date "2037-06-15"}+ (premium-features/transform-stats))))++ (testing "with :hosting :transforms-basic :writable-connection :transforms-python,+ both mbql and python runs meter as advanced"+ (mt/with-premium-features #{:hosting :transforms-basic :writable-connection :transforms-python}+ (run! mbql-id frozen-yesterday)+ (run! python-id frozen-yesterday)+ (run! mbql-id frozen-today)+ (run! python-id frozen-today))+ (is (= {:transform-basic-runs 1+ :transform-advanced-runs 2+ :transform-usage-date "2037-06-14"+ :transform-rolling-basic-runs 1+ :transform-rolling-advanced-runs 2+ :transform-rolling-usage-date "2037-06-15"}+ (premium-features/transform-stats)))))))))))diff --git a/resources/migrations/060/20260415_transform_run_metered_as.yaml b/resources/migrations/060/20260415_transform_run_metered_as.yamlnew file mode 100644index 00000000000..262cee9c688--- /dev/null+++ b/resources/migrations/060/20260415_transform_run_metered_as.yaml@@ -0,0 +1,22 @@+## Record the meter bucket each transform run counts toward at run start time.++databaseChangeLog:+ - logicalFilePath: migrations/060_update_migrations.yaml+ - changeSet:+ id: v60.2026-04-15T12:00:00+ author: filipesilva+ comment: Add metered_as column to transform_run for per-run billing attribution+ changes:+ - addColumn:+ tableName: transform_run+ columns:+ - column:+ name: metered_as+ type: varchar(50)+ remarks: The meter bucket this run counts toward (transform-basic, transform-advanced, or null)+ constraints:+ nullable: true+ rollback:+ - dropColumn:+ tableName: transform_run+ columnName: metered_asdiff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.cljindex e7c78c0d799..29854061ad8 100644--- a/src/metabase/premium_features/core.clj+++ b/src/metabase/premium_features/core.clj@@ -36,6 +36,7 @@TokenStatusclear-cache!token-check-url+ transform-metered-astransform-stats](metabase.premium-features.settingsdiff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex b666d1f4646..704573749fb 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -129,6 +129,14 @@:transform-rolling-advanced-runs 0:transform-rolling-usage-date (today)})+(defenterprise transform-metered-as+ "Return the meter bucket a new transform run of the given source-type counts toward,+ based on the instance's current premium features. Returns nil when the run is not metered+ (including all OSS runs, since premium features are never present there)."+ metabase-enterprise.transforms.core+ [_source-type]+ nil)+(defn metering-stats"Collect metering statistics for billing purposes. Used by both token check and metering task. "[]diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.cljindex 865a6c053b7..35ee0e8cfb3 100644--- a/src/metabase/transforms/models/transform_run.clj+++ b/src/metabase/transforms/models/transform_run.clj@@ -5,6 +5,7 @@[metabase.collections.models.collection.root :as collection.root][metabase.events.core :as events][metabase.models.interface :as mi]+ [metabase.premium-features.core :as premium-features][metabase.query-processor.parameters.dates :as params.dates][metabase.transforms.models.transform-run-cancelation :as cancel][metabase.util :as u]@@ -62,14 +63,16 @@([transform-id](start-run! transform-id {}))([transform-id properties]- (let [transform (t2/select-one [:model/Transform :name :entity_id] :id transform-id)+ (let [transform (t2/select-one [:model/Transform :name :entity_id :source_type] :id transform-id)+ metered-as (premium-features/transform-metered-as (:source_type transform))run (t2/insert-returning-instance! :model/TransformRun(assoc properties:transform_id transform-id:transform_name (:name transform):transform_entity_id (:entity_id transform):status :started- :is_active true))]+ :is_active true+ :metered_as metered-as))];; Pass user_id to the event so audit log properly attributes the run(events/publish-event! :event/transform-run-start(cond-> {:object run}diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 94bdc358fc3..cd4e83bf1b3 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -67,7 +67,8 @@[:transform_entity_id {:optional true} [:maybe :string]][:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]][:checkpoint_lo_value {:optional true} [:maybe :string]]- [:checkpoint_hi_value {:optional true} [:maybe :string]]])+ [:checkpoint_hi_value {:optional true} [:maybe :string]]+ [:metered_as {:optional true} [:maybe :string]]])(def ^:private TransformResponse[:map {:closed true}@@ -112,6 +113,7 @@[:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]][:checkpoint_lo_value {:optional true} [:maybe :string]][:checkpoint_hi_value {:optional true} [:maybe :string]]+ [:metered_as {:optional true} [:maybe :string]];; Transform can have id/name when exists, or be nil when deleted[:transform {:optional true} [:maybe [:map {:closed true}[:id {:optional true} pos-int?]