Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-1872: Change transform metering to tier-based keys (#70366)

Ngoc Khuate822288497ec
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
index 0cacb5bd971..627eae57e13 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
@@ -7,7 +7,7 @@
But we need a 'real' module, because all tests must belong to a module."
(:require
[java-time.api :as t]
- [metabase.premium-features.core :refer [defenterprise]]
+ [metabase.premium-features.core :as premium-features :refer [defenterprise]]
[toucan2.core :as t2]))
(defenterprise transform-stats
@@ -15,20 +15,22 @@
Rolling stats are included under :transform-rolling (up to date totals for today)"
:feature :none
[]
- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))
- yesterday-utc (t/minus today-utc (t/days 1))
- count-runs (fn [source-types date]
- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]
- :from [[:transform_run :r]]
- :join [[:transform :t] [:= :t.id :r.transform_id]]
- :where [:and
- [:= :r.status "succeeded"]
- [:in :t.source_type source-types]
- [:= [:cast :r.end_time :date] [:cast date :date]]]}))
- 0))]
- {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)
- :transform-python-runs (count-runs ["python"] yesterday-utc)
- :transform-usage-date (str (t/local-date yesterday-utc))
- :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)
- :transform-rolling-python-runs (count-runs ["python"] today-utc)
- :transform-rolling-usage-date (str (t/local-date today-utc))}))
+ (let [today-utc (t/offset-date-time (t/zone-offset "+00"))
+ yesterday-utc (t/minus today-utc (t/days 1))
+ count-all-runs (fn [date]
+ (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]
+ :from [[:transform_run :r]]
+ :where [:and
+ [:= :r.status "succeeded"]
+ [:= [:cast :r.end_time :date] [:cast date :date]]]}))
+ 0))
+ advanced? (and (premium-features/has-feature? :transforms)
+ (premium-features/has-feature? :transforms-python))
+ yesterday-runs (count-all-runs yesterday-utc)
+ today-runs (count-all-runs today-utc)]
+ {:transform-basic-runs (if advanced? 0 yesterday-runs)
+ :transform-advanced-runs (if advanced? yesterday-runs 0)
+ :transform-usage-date (str (t/local-date yesterday-utc))
+ :transform-rolling-basic-runs (if advanced? 0 today-runs)
+ :transform-rolling-advanced-runs (if advanced? today-runs 0)
+ :transform-rolling-usage-date (str (t/local-date today-utc))}))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
index 201f1f0d065..550ab517d74 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
@@ -12,52 +12,52 @@
(use-fixtures :once (fixtures/initialize :db))
-(deftest transform-stats-rolling-to-yesterday-test
- (testing "completed runs appear in rolling stats, then move to yesterday's stats when end_time is backdated"
+(deftest transform-stats-tier-based-metering-test
+ (testing "all runs are bucketed into basic or advanced based on the transforms-python feature flag"
(mt/with-temp [:model/Transform {native-id :id} {}
:model/Transform {python-id :id} {:source {:type "python"
:source-database (mt/id)}
:target {:type "table"
:name (str "test_python_" (random-uuid))
:database (mt/id)}}]
- (let [stats-before (premium-features/transform-stats)
- native-before (:transform-native-runs stats-before)
- rolling-native-before (:transform-rolling-native-runs stats-before)
- python-before (:transform-python-runs stats-before)
- rolling-python-before (:transform-rolling-python-runs stats-before)
- yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]
-
- (testing "native"
+ (let [stats-before (premium-features/transform-stats)
+ basic-before (:transform-basic-runs stats-before)
+ advanced-before (:transform-advanced-runs stats-before)
+ rolling-basic-before (:transform-rolling-basic-runs stats-before)
+ rolling-advanced-before (:transform-rolling-advanced-runs stats-before)
+ yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]
+
+ (testing "without advanced add-on, all runs go to basic-runs"
+ ;; run a native transform
(let [{run-id :id} (transform-run/start-run! native-id {:run_method "manual"})]
(transform-run/succeed-started-run! run-id)
- (testing "completed run appears in rolling stats only"
+ (testing "completed run appears in rolling-basic-runs"
(let [stats (premium-features/transform-stats)]
- (is (= (inc rolling-native-before) (:transform-rolling-native-runs stats)))
- (is (= native-before (:transform-native-runs stats)))))
+ (is (= (inc rolling-basic-before) (:transform-rolling-basic-runs stats)))
+ (is (= rolling-advanced-before (:transform-rolling-advanced-runs stats)))
+ (is (= basic-before (:transform-basic-runs stats)))))
- (testing "a run for yesterday is visible under yesterday's stats only"
+ (testing "backdated run appears in basic-runs"
(t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})
(let [stats (premium-features/transform-stats)]
- (is (= rolling-native-before (:transform-rolling-native-runs stats)))
- (is (= (inc native-before) (:transform-native-runs stats)))))))
-
- (testing "python stats not modified by native runs"
- (let [stats (premium-features/transform-stats)]
- (is (= python-before (:transform-python-runs stats)))
- (is (= rolling-python-before (:transform-rolling-python-runs stats)))))
-
- (testing "python"
- (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]
- (transform-run/succeed-started-run! run-id)
-
- (testing "completed run appears in rolling stats only"
- (let [stats (premium-features/transform-stats)]
- (is (= (inc rolling-python-before) (:transform-rolling-python-runs stats)))
- (is (= python-before (:transform-python-runs stats)))))
-
- (testing "a run for yesterday is visible under yesterday's state only"
- (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})
- (let [stats (premium-features/transform-stats)]
- (is (= rolling-python-before (:transform-rolling-python-runs stats)))
- (is (= (inc python-before) (:transform-python-runs stats)))))))))))
+ (is (= rolling-basic-before (:transform-rolling-basic-runs stats)))
+ (is (= (inc basic-before) (:transform-basic-runs stats)))
+ (is (= advanced-before (:transform-advanced-runs stats)))))))
+
+ (testing "with advanced add-on, all runs go to advanced-runs"
+ (mt/with-premium-features #{:transforms :transforms-python}
+ ;; run a python transform
+ (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]
+ (transform-run/succeed-started-run! run-id)
+
+ (testing "completed run appears in rolling-advanced-runs"
+ (let [stats (premium-features/transform-stats)]
+ (is (= 0 (:transform-rolling-basic-runs stats)))
+ (is (pos? (:transform-rolling-advanced-runs stats)))))
+
+ (testing "backdated run appears in advanced-runs"
+ (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})
+ (let [stats (premium-features/transform-stats)]
+ (is (= 0 (:transform-basic-runs stats)))
+ (is (pos? (:transform-advanced-runs stats))))))))))))
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index fb59edec0e9..372286b9af8 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -130,12 +130,12 @@
"Stats for Transforms"
metabase-enterprise.transforms.core
[]
- {:transform-native-runs 0
- :transform-python-runs 0
- :transform-usage-date (yesterday)
- :transform-rolling-native-runs 0
- :transform-rolling-python-runs 0
- :transform-rolling-usage-date (today)})
+ {:transform-basic-runs 0
+ :transform-advanced-runs 0
+ :transform-usage-date (yesterday)
+ :transform-rolling-basic-runs 0
+ :transform-rolling-advanced-runs 0
+ :transform-rolling-usage-date (today)})
(defn metering-stats
"Collect metering statistics for billing purposes. Used by both token check and metering task. "
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index 9dce8efb814..813280d8245 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -1,7 +1,6 @@
(ns metabase.transforms.core
"API namespace for the `metabase.transforms` module."
(:require
- [java-time.api :as t]
[metabase.models.transforms.transform]
[metabase.models.transforms.transform-job]
[metabase.models.transforms.transform-run]
@@ -16,8 +15,7 @@
[metabase.transforms.schedule]
[metabase.transforms.settings]
[metabase.transforms.util]
- [potemkin :as p]
- [toucan2.core :as t2]))
+ [potemkin :as p]))
(p/import-vars
[metabase.transforms.settings
@@ -74,25 +72,3 @@
[metabase.models.transforms.transform-tag
tag-name-exists?
tag-name-exists-excluding?])
-
-(defn transform-stats
- "Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.
- Rolling stats are included under :transform-rolling (up to date totals for today)"
- []
- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))
- yesterday-utc (t/minus today-utc (t/days 1))
- count-runs (fn [source-types date]
- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]
- :from [[:transform_run :r]]
- :join [[:transform :t] [:= :t.id :r.transform_id]]
- :where [:and
- [:= :r.status "succeeded"]
- [:in :t.source_type source-types]
- [:= [:cast :r.end_time :date] [:cast date :date]]]}))
- 0))]
- {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)
- :transform-python-runs (count-runs ["python"] yesterday-utc)
- :transform-usage-date (str (t/local-date yesterday-utc))
- :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)
- :transform-rolling-python-runs (count-runs ["python"] today-utc)
- :transform-rolling-usage-date (str (t/local-date today-utc))}))