The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-1872: Change transform metering to tier-based keys (#70366)
Ngoc Khuate822288497ec
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.cljindex 0cacb5bd971..627eae57e13 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj@@ -7,7 +7,7 @@But we need a 'real' module, because all tests must belong to a module."(:require[java-time.api :as t]- [metabase.premium-features.core :refer [defenterprise]]+ [metabase.premium-features.core :as premium-features :refer [defenterprise]][toucan2.core :as t2]))(defenterprise transform-stats@@ -15,20 +15,22 @@Rolling stats are included under :transform-rolling (up to date totals for today)":feature :none[]- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))- yesterday-utc (t/minus today-utc (t/days 1))- count-runs (fn [source-types date]- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]- :from [[:transform_run :r]]- :join [[:transform :t] [:= :t.id :r.transform_id]]- :where [:and- [:= :r.status "succeeded"]- [:in :t.source_type source-types]- [:= [:cast :r.end_time :date] [:cast date :date]]]}))- 0))]- {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)- :transform-python-runs (count-runs ["python"] yesterday-utc)- :transform-usage-date (str (t/local-date yesterday-utc))- :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)- :transform-rolling-python-runs (count-runs ["python"] today-utc)- :transform-rolling-usage-date (str (t/local-date today-utc))}))+ (let [today-utc (t/offset-date-time (t/zone-offset "+00"))+ yesterday-utc (t/minus today-utc (t/days 1))+ count-all-runs (fn [date]+ (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]+ :from [[:transform_run :r]]+ :where [:and+ [:= :r.status "succeeded"]+ [:= [:cast :r.end_time :date] [:cast date :date]]]}))+ 0))+ advanced? (and (premium-features/has-feature? :transforms)+ (premium-features/has-feature? :transforms-python))+ yesterday-runs (count-all-runs yesterday-utc)+ today-runs (count-all-runs today-utc)]+ {:transform-basic-runs (if advanced? 0 yesterday-runs)+ :transform-advanced-runs (if advanced? yesterday-runs 0)+ :transform-usage-date (str (t/local-date yesterday-utc))+ :transform-rolling-basic-runs (if advanced? 0 today-runs)+ :transform-rolling-advanced-runs (if advanced? today-runs 0)+ :transform-rolling-usage-date (str (t/local-date today-utc))}))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/core_test.cljindex 201f1f0d065..550ab517d74 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj@@ -12,52 +12,52 @@(use-fixtures :once (fixtures/initialize :db))-(deftest transform-stats-rolling-to-yesterday-test- (testing "completed runs appear in rolling stats, then move to yesterday's stats when end_time is backdated"+(deftest transform-stats-tier-based-metering-test+ (testing "all runs are bucketed into basic or advanced based on the transforms-python feature flag"(mt/with-temp [:model/Transform {native-id :id} {}:model/Transform {python-id :id} {:source {:type "python":source-database (mt/id)}:target {:type "table":name (str "test_python_" (random-uuid)):database (mt/id)}}]- (let [stats-before (premium-features/transform-stats)- native-before (:transform-native-runs stats-before)- rolling-native-before (:transform-rolling-native-runs stats-before)- python-before (:transform-python-runs stats-before)- rolling-python-before (:transform-rolling-python-runs stats-before)- yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]-- (testing "native"+ (let [stats-before (premium-features/transform-stats)+ basic-before (:transform-basic-runs stats-before)+ advanced-before (:transform-advanced-runs stats-before)+ rolling-basic-before (:transform-rolling-basic-runs stats-before)+ rolling-advanced-before (:transform-rolling-advanced-runs stats-before)+ yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]++ (testing "without advanced add-on, all runs go to basic-runs"+ ;; run a native transform(let [{run-id :id} (transform-run/start-run! native-id {:run_method "manual"})](transform-run/succeed-started-run! run-id)- (testing "completed run appears in rolling stats only"+ (testing "completed run appears in rolling-basic-runs"(let [stats (premium-features/transform-stats)]- (is (= (inc rolling-native-before) (:transform-rolling-native-runs stats)))- (is (= native-before (:transform-native-runs stats)))))+ (is (= (inc rolling-basic-before) (:transform-rolling-basic-runs stats)))+ (is (= rolling-advanced-before (:transform-rolling-advanced-runs stats)))+ (is (= basic-before (:transform-basic-runs stats)))))- (testing "a run for yesterday is visible under yesterday's stats only"+ (testing "backdated run appears in basic-runs"(t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})(let [stats (premium-features/transform-stats)]- (is (= rolling-native-before (:transform-rolling-native-runs stats)))- (is (= (inc native-before) (:transform-native-runs stats)))))))-- (testing "python stats not modified by native runs"- (let [stats (premium-features/transform-stats)]- (is (= python-before (:transform-python-runs stats)))- (is (= rolling-python-before (:transform-rolling-python-runs stats)))))-- (testing "python"- (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]- (transform-run/succeed-started-run! run-id)-- (testing "completed run appears in rolling stats only"- (let [stats (premium-features/transform-stats)]- (is (= (inc rolling-python-before) (:transform-rolling-python-runs stats)))- (is (= python-before (:transform-python-runs stats)))))-- (testing "a run for yesterday is visible under yesterday's state only"- (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})- (let [stats (premium-features/transform-stats)]- (is (= rolling-python-before (:transform-rolling-python-runs stats)))- (is (= (inc python-before) (:transform-python-runs stats)))))))))))+ (is (= rolling-basic-before (:transform-rolling-basic-runs stats)))+ (is (= (inc basic-before) (:transform-basic-runs stats)))+ (is (= advanced-before (:transform-advanced-runs stats)))))))++ (testing "with advanced add-on, all runs go to advanced-runs"+ (mt/with-premium-features #{:transforms :transforms-python}+ ;; run a python transform+ (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]+ (transform-run/succeed-started-run! run-id)++ (testing "completed run appears in rolling-advanced-runs"+ (let [stats (premium-features/transform-stats)]+ (is (= 0 (:transform-rolling-basic-runs stats)))+ (is (pos? (:transform-rolling-advanced-runs stats)))))++ (testing "backdated run appears in advanced-runs"+ (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})+ (let [stats (premium-features/transform-stats)]+ (is (= 0 (:transform-basic-runs stats)))+ (is (pos? (:transform-advanced-runs stats))))))))))))diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex fb59edec0e9..372286b9af8 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -130,12 +130,12 @@"Stats for Transforms"metabase-enterprise.transforms.core[]- {:transform-native-runs 0- :transform-python-runs 0- :transform-usage-date (yesterday)- :transform-rolling-native-runs 0- :transform-rolling-python-runs 0- :transform-rolling-usage-date (today)})+ {:transform-basic-runs 0+ :transform-advanced-runs 0+ :transform-usage-date (yesterday)+ :transform-rolling-basic-runs 0+ :transform-rolling-advanced-runs 0+ :transform-rolling-usage-date (today)})(defn metering-stats"Collect metering statistics for billing purposes. Used by both token check and metering task. "diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 9dce8efb814..813280d8245 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -1,7 +1,6 @@(ns metabase.transforms.core"API namespace for the `metabase.transforms` module."(:require- [java-time.api :as t][metabase.models.transforms.transform][metabase.models.transforms.transform-job][metabase.models.transforms.transform-run]@@ -16,8 +15,7 @@[metabase.transforms.schedule][metabase.transforms.settings][metabase.transforms.util]- [potemkin :as p]- [toucan2.core :as t2]))+ [potemkin :as p]))(p/import-vars[metabase.transforms.settings@@ -74,25 +72,3 @@[metabase.models.transforms.transform-tagtag-name-exists?tag-name-exists-excluding?])--(defn transform-stats- "Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.- Rolling stats are included under :transform-rolling (up to date totals for today)"- []- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))- yesterday-utc (t/minus today-utc (t/days 1))- count-runs (fn [source-types date]- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]- :from [[:transform_run :r]]- :join [[:transform :t] [:= :t.id :r.transform_id]]- :where [:and- [:= :r.status "succeeded"]- [:in :t.source_type source-types]- [:= [:cast :r.end_time :date] [:cast date :date]]]}))- 0))]- {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)- :transform-python-runs (count-runs ["python"] yesterday-utc)- :transform-usage-date (str (t/local-date yesterday-utc))- :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)- :transform-rolling-python-runs (count-runs ["python"] today-utc)- :transform-rolling-usage-date (str (t/local-date today-utc))}))