Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix: fix transform run metering aggregation (#72632)

Filipe Silva9571b2b4c3dd
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
index 92c1bc4cf62..63d9326da8c 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
@@ -10,27 +10,50 @@
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
[toucan2.core :as t2]))
+(defenterprise transform-metered-as
+ "Return the meter bucket a new transform run of the given source-type counts toward,
+ based on the instance's current premium features. Returns nil when the run is not metered."
+ :feature :none
+ [source-type]
+ (let [hosted? (premium-features/has-feature? :hosting)
+ basic? (premium-features/has-feature? :transforms-basic)
+ writable? (premium-features/has-feature? :writable-connection)
+ python? (premium-features/has-feature? :transforms-python)]
+ (case (keyword source-type)
+ (:native :mbql) (cond
+ ;; These transforms are metered differently based on plan and addons status:
+ ;; - the advanced transforms addon adds the writable-connection feature, which
+ ;; augments what they can do and meters them as advanced
+ (and basic? writable?) "transform-advanced"
+ ;; - hosted instances with basic transforms get these metered as basic
+ (and basic? hosted?) "transform-basic"
+ ;; - self-hosted customers without the advanced add-on aren't metered for these at all
+ :else nil)
+ :python (when python? "transform-advanced")
+ nil)))
+
(defenterprise transform-stats
"Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.
- Rolling stats are included under :transform-rolling (up to date totals for today)"
+ Rolling stats are included under :transform-rolling (up to date totals for today).
+
+ Runs are bucketed by the :metered_as column set at `start-run!` time."
:feature :none
[]
- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))
- yesterday-utc (t/minus today-utc (t/days 1))
- count-all-runs (fn [date]
- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]
- :from [[:transform_run :r]]
- :where [:and
- [:= :r.status "succeeded"]
- [:= [:cast :r.end_time :date] [:cast date :date]]]}))
- 0))
- advanced? (and (premium-features/has-feature? :transforms-basic)
- (premium-features/has-feature? :transforms-python))
- yesterday-runs (count-all-runs yesterday-utc)
- today-runs (count-all-runs today-utc)]
- {:transform-basic-runs (if advanced? 0 yesterday-runs)
- :transform-advanced-runs (if advanced? yesterday-runs 0)
- :transform-usage-date (str (t/local-date yesterday-utc))
- :transform-rolling-basic-runs (if advanced? 0 today-runs)
- :transform-rolling-advanced-runs (if advanced? today-runs 0)
- :transform-rolling-usage-date (str (t/local-date today-utc))}))
+ (let [today-utc (t/offset-date-time (t/zone-offset "+00"))
+ yesterday-utc (t/minus today-utc (t/days 1))
+ counts-for (fn [date]
+ (into {} (map (juxt :metered_as :cnt))
+ (t2/query {:select [:r.metered_as [[:count :r.id] :cnt]]
+ :from [[:transform_run :r]]
+ :where [:and
+ [:= :r.status "succeeded"]
+ [:= [:cast :r.end_time :date] [:cast date :date]]]
+ :group-by [:r.metered_as]})))
+ yesterday (counts-for yesterday-utc)
+ today (counts-for today-utc)]
+ {:transform-basic-runs (get yesterday "transform-basic" 0)
+ :transform-advanced-runs (get yesterday "transform-advanced" 0)
+ :transform-usage-date (str (t/local-date yesterday-utc))
+ :transform-rolling-basic-runs (get today "transform-basic" 0)
+ :transform-rolling-advanced-runs (get today "transform-advanced" 0)
+ :transform-rolling-usage-date (str (t/local-date today-utc))}))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
index db1d607fb73..86f5e0b0b3f 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
@@ -2,6 +2,8 @@
(:require
[clojure.test :refer :all]
[java-time.api :as t]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.premium-features.core :as premium-features]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -12,52 +14,90 @@
(use-fixtures :once (fixtures/initialize :db))
-(deftest transform-stats-tier-based-metering-test
- (testing "all runs are bucketed into basic or advanced based on the transforms-python feature flag"
- (mt/with-temp [:model/Transform {native-id :id} {}
- :model/Transform {python-id :id} {:source {:type "python"
- :source-database (mt/id)}
- :target {:type "table"
- :name (str "test_python_" (random-uuid))
- :database (mt/id)}}]
- (let [stats-before (premium-features/transform-stats)
- basic-before (:transform-basic-runs stats-before)
- advanced-before (:transform-advanced-runs stats-before)
- rolling-basic-before (:transform-rolling-basic-runs stats-before)
- rolling-advanced-before (:transform-rolling-advanced-runs stats-before)
- yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]
-
- (testing "without advanced add-on, all runs go to basic-runs"
- ;; run a native transform
- (let [{run-id :id} (transform-run/start-run! native-id {:run_method "manual"})]
- (transform-run/succeed-started-run! run-id)
-
- (testing "completed run appears in rolling-basic-runs"
- (let [stats (premium-features/transform-stats)]
- (is (= (inc rolling-basic-before) (:transform-rolling-basic-runs stats)))
- (is (= rolling-advanced-before (:transform-rolling-advanced-runs stats)))
- (is (= basic-before (:transform-basic-runs stats)))))
-
- (testing "backdated run appears in basic-runs"
- (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})
- (let [stats (premium-features/transform-stats)]
- (is (= rolling-basic-before (:transform-rolling-basic-runs stats)))
- (is (= (inc basic-before) (:transform-basic-runs stats)))
- (is (= advanced-before (:transform-advanced-runs stats)))))))
-
- (testing "with advanced add-on, all runs go to advanced-runs"
- (mt/with-premium-features #{:transforms-basic :transforms-python}
- ;; run a python transform
- (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]
- (transform-run/succeed-started-run! run-id)
-
- (testing "completed run appears in rolling-advanced-runs"
- (let [stats (premium-features/transform-stats)]
- (is (= 0 (:transform-rolling-basic-runs stats)))
- (is (pos? (:transform-rolling-advanced-runs stats)))))
-
- (testing "backdated run appears in advanced-runs"
- (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})
- (let [stats (premium-features/transform-stats)]
- (is (= 0 (:transform-basic-runs stats)))
- (is (pos? (:transform-advanced-runs stats))))))))))))
+(deftest transform-metered-as-test
+ (mt/with-premium-features #{:transforms-basic :writable-connection}
+ (is (= "transform-advanced"
+ (premium-features/transform-metered-as :native)
+ (premium-features/transform-metered-as :mbql))))
+
+ (mt/with-premium-features #{:hosting :transforms-basic}
+ (is (= "transform-basic"
+ (premium-features/transform-metered-as :native)
+ (premium-features/transform-metered-as :mbql))))
+
+ (mt/with-premium-features #{:transforms-python}
+ (is (= "transform-advanced"
+ (premium-features/transform-metered-as :python))))
+
+ (mt/with-premium-features #{}
+ (is (= nil
+ (premium-features/transform-metered-as :native)
+ (premium-features/transform-metered-as :mbql))))
+
+ (mt/with-premium-features #{:hosting :transforms-basic :transforms-python :writable-connection}
+ (is (= nil
+ (premium-features/transform-metered-as nil)
+ (premium-features/transform-metered-as :something-else)))))
+
+(deftest transform-stats-aggregation-test
+ (testing "transform-stats buckets runs by the :metered_as captured at start-run! time"
+ (let [frozen-today (t/offset-date-time 2037 6 15 12 0 0 0 (t/zone-offset "+00"))
+ frozen-yesterday (t/minus frozen-today (t/days 1))
+ mp (mt/metadata-provider)]
+ (with-redefs [t/offset-date-time (constantly frozen-today)]
+ (mt/with-model-cleanup [:model/TransformRun]
+ (mt/with-temp [:model/Transform {mbql-id :id}
+ {:source {:type "query"
+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}
+ :target {:type "table"
+ :name (str "test_mbql_" (random-uuid))
+ :database (mt/id)}}
+ :model/Transform {python-id :id}
+ {:source {:type "python"
+ :source-database (mt/id)}
+ :target {:type "table"
+ :name (str "test_python_" (random-uuid))
+ :database (mt/id)}}]
+ (let [run! (fn [transform-id end-time]
+ (let [{id :id} (transform-run/start-run! transform-id {:run_method "manual"})]
+ (transform-run/succeed-started-run! id)
+ (t2/update! :model/TransformRun :id id {:end_time end-time})))]
+
+ (testing "with no premium features, mbql runs are not metered"
+ (mt/with-premium-features #{}
+ (run! mbql-id frozen-yesterday)
+ (run! mbql-id frozen-today))
+ (is (= {:transform-basic-runs 0
+ :transform-advanced-runs 0
+ :transform-usage-date "2037-06-14"
+ :transform-rolling-basic-runs 0
+ :transform-rolling-advanced-runs 0
+ :transform-rolling-usage-date "2037-06-15"}
+ (premium-features/transform-stats))))
+
+ (testing "with :hosting :transforms-basic, mbql runs meter as basic"
+ (mt/with-premium-features #{:hosting :transforms-basic}
+ (run! mbql-id frozen-yesterday)
+ (run! mbql-id frozen-today))
+ (is (= {:transform-basic-runs 1
+ :transform-advanced-runs 0
+ :transform-usage-date "2037-06-14"
+ :transform-rolling-basic-runs 1
+ :transform-rolling-advanced-runs 0
+ :transform-rolling-usage-date "2037-06-15"}
+ (premium-features/transform-stats))))
+
+ (testing "with :hosting :transforms-basic :writable-connection :transforms-python,
+ both mbql and python runs meter as advanced"
+ (mt/with-premium-features #{:hosting :transforms-basic :writable-connection :transforms-python}
+ (run! mbql-id frozen-yesterday)
+ (run! python-id frozen-yesterday)
+ (run! mbql-id frozen-today)
+ (run! python-id frozen-today))
+ (is (= {:transform-basic-runs 1
+ :transform-advanced-runs 2
+ :transform-usage-date "2037-06-14"
+ :transform-rolling-basic-runs 1
+ :transform-rolling-advanced-runs 2
+ :transform-rolling-usage-date "2037-06-15"}
+ (premium-features/transform-stats)))))))))))
diff --git a/resources/migrations/060/20260415_transform_run_metered_as.yaml b/resources/migrations/060/20260415_transform_run_metered_as.yaml
new file mode 100644
index 00000000000..262cee9c688
--- /dev/null
+++ b/resources/migrations/060/20260415_transform_run_metered_as.yaml
@@ -0,0 +1,22 @@
+## Record the meter bucket each transform run counts toward at run start time.
+
+databaseChangeLog:
+ - logicalFilePath: migrations/060_update_migrations.yaml
+ - changeSet:
+ id: v60.2026-04-15T12:00:00
+ author: filipesilva
+ comment: Add metered_as column to transform_run for per-run billing attribution
+ changes:
+ - addColumn:
+ tableName: transform_run
+ columns:
+ - column:
+ name: metered_as
+ type: varchar(50)
+ remarks: The meter bucket this run counts toward (transform-basic, transform-advanced, or null)
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: transform_run
+ columnName: metered_as
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index e7c78c0d799..29854061ad8 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -36,6 +36,7 @@
TokenStatus
clear-cache!
token-check-url
+ transform-metered-as
transform-stats]
(metabase.premium-features.settings
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index b666d1f4646..704573749fb 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -129,6 +129,14 @@
:transform-rolling-advanced-runs 0
:transform-rolling-usage-date (today)})
+(defenterprise transform-metered-as
+ "Return the meter bucket a new transform run of the given source-type counts toward,
+ based on the instance's current premium features. Returns nil when the run is not metered
+ (including all OSS runs, since premium features are never present there)."
+ metabase-enterprise.transforms.core
+ [_source-type]
+ nil)
+
(defn metering-stats
"Collect metering statistics for billing purposes. Used by both token check and metering task. "
[]
diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.clj
index 865a6c053b7..35ee0e8cfb3 100644
--- a/src/metabase/transforms/models/transform_run.clj
+++ b/src/metabase/transforms/models/transform_run.clj
@@ -5,6 +5,7 @@
[metabase.collections.models.collection.root :as collection.root]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
+ [metabase.premium-features.core :as premium-features]
[metabase.query-processor.parameters.dates :as params.dates]
[metabase.transforms.models.transform-run-cancelation :as cancel]
[metabase.util :as u]
@@ -62,14 +63,16 @@
([transform-id]
(start-run! transform-id {}))
([transform-id properties]
- (let [transform (t2/select-one [:model/Transform :name :entity_id] :id transform-id)
+ (let [transform (t2/select-one [:model/Transform :name :entity_id :source_type] :id transform-id)
+ metered-as (premium-features/transform-metered-as (:source_type transform))
run (t2/insert-returning-instance! :model/TransformRun
(assoc properties
:transform_id transform-id
:transform_name (:name transform)
:transform_entity_id (:entity_id transform)
:status :started
- :is_active true))]
+ :is_active true
+ :metered_as metered-as))]
;; Pass user_id to the event so audit log properly attributes the run
(events/publish-event! :event/transform-run-start
(cond-> {:object run}
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 94bdc358fc3..cd4e83bf1b3 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -67,7 +67,8 @@
[:transform_entity_id {:optional true} [:maybe :string]]
[:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
[:checkpoint_lo_value {:optional true} [:maybe :string]]
- [:checkpoint_hi_value {:optional true} [:maybe :string]]])
+ [:checkpoint_hi_value {:optional true} [:maybe :string]]
+ [:metered_as {:optional true} [:maybe :string]]])
(def ^:private TransformResponse
[:map {:closed true}
@@ -112,6 +113,7 @@
[:checkpoint_filter_field_id {:optional true} [:maybe pos-int?]]
[:checkpoint_lo_value {:optional true} [:maybe :string]]
[:checkpoint_hi_value {:optional true} [:maybe :string]]
+ [:metered_as {:optional true} [:maybe :string]]
;; Transform can have id/name when exists, or be nil when deleted
[:transform {:optional true} [:maybe [:map {:closed true}
[:id {:optional true} pos-int?]