Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Move transform metering back into EE module (#69840)

Dan Stoneacf4530cc6c3
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 1eaf8981e41..6923c670f38 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2435,7 +2435,7 @@
enterprise/transforms
{:team "Querying"
:api #{}
- :uses #{}}
+ :uses #{premium-features}}
enterprise/transforms-python
{:team "Querying"
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
index 4522dddfb89..0cacb5bd971 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
@@ -1,9 +1,34 @@
(ns metabase-enterprise.transforms.core
- "Enterprise module placeholder for transforms.
+ "Enterprise module for transforms used for both tests and metering.
Transforms are weird: they have different behavior on OSS vs EE. We need to test this, so we need tests
in `enterprise/backend/test/metabase_enterprise/transforms/api_test.clj`.
- But we need a 'real' module, because all tests must belong to a module.
+ But we need a 'real' module, because all tests must belong to a module."
+ (:require
+ [java-time.api :as t]
+ [metabase.premium-features.core :refer [defenterprise]]
+ [toucan2.core :as t2]))
- Hence, this placeholder.")
+(defenterprise transform-stats
+ "Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.
+ Rolling stats are included under :transform-rolling (up to date totals for today)"
+ :feature :none
+ []
+ (let [today-utc (t/offset-date-time (t/zone-offset "+00"))
+ yesterday-utc (t/minus today-utc (t/days 1))
+ count-runs (fn [source-types date]
+ (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]
+ :from [[:transform_run :r]]
+ :join [[:transform :t] [:= :t.id :r.transform_id]]
+ :where [:and
+ [:= :r.status "succeeded"]
+ [:in :t.source_type source-types]
+ [:= [:cast :r.end_time :date] [:cast date :date]]]}))
+ 0))]
+ {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)
+ :transform-python-runs (count-runs ["python"] yesterday-utc)
+ :transform-usage-date (str (t/local-date yesterday-utc))
+ :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)
+ :transform-rolling-python-runs (count-runs ["python"] today-utc)
+ :transform-rolling-usage-date (str (t/local-date today-utc))}))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
new file mode 100644
index 00000000000..201f1f0d065
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj
@@ -0,0 +1,63 @@
+(ns metabase-enterprise.transforms.core-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.models.transforms.transform-run :as transform-run]
+ [metabase.premium-features.core :as premium-features]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db))
+
+(deftest transform-stats-rolling-to-yesterday-test
+ (testing "completed runs appear in rolling stats, then move to yesterday's stats when end_time is backdated"
+ (mt/with-temp [:model/Transform {native-id :id} {}
+ :model/Transform {python-id :id} {:source {:type "python"
+ :source-database (mt/id)}
+ :target {:type "table"
+ :name (str "test_python_" (random-uuid))
+ :database (mt/id)}}]
+ (let [stats-before (premium-features/transform-stats)
+ native-before (:transform-native-runs stats-before)
+ rolling-native-before (:transform-rolling-native-runs stats-before)
+ python-before (:transform-python-runs stats-before)
+ rolling-python-before (:transform-rolling-python-runs stats-before)
+ yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]
+
+ (testing "native"
+ (let [{run-id :id} (transform-run/start-run! native-id {:run_method "manual"})]
+ (transform-run/succeed-started-run! run-id)
+
+ (testing "completed run appears in rolling stats only"
+ (let [stats (premium-features/transform-stats)]
+ (is (= (inc rolling-native-before) (:transform-rolling-native-runs stats)))
+ (is (= native-before (:transform-native-runs stats)))))
+
+ (testing "a run for yesterday is visible under yesterday's stats only"
+ (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})
+ (let [stats (premium-features/transform-stats)]
+ (is (= rolling-native-before (:transform-rolling-native-runs stats)))
+ (is (= (inc native-before) (:transform-native-runs stats)))))))
+
+ (testing "python stats not modified by native runs"
+ (let [stats (premium-features/transform-stats)]
+ (is (= python-before (:transform-python-runs stats)))
+ (is (= rolling-python-before (:transform-rolling-python-runs stats)))))
+
+ (testing "python"
+ (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]
+ (transform-run/succeed-started-run! run-id)
+
+ (testing "completed run appears in rolling stats only"
+ (let [stats (premium-features/transform-stats)]
+ (is (= (inc rolling-python-before) (:transform-rolling-python-runs stats)))
+ (is (= python-before (:transform-python-runs stats)))))
+
+ (testing "a run for yesterday is visible under yesterday's state only"
+ (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})
+ (let [stats (premium-features/transform-stats)]
+ (is (= rolling-python-before (:transform-rolling-python-runs stats)))
+ (is (= (inc python-before) (:transform-python-runs stats)))))))))))
diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.clj
index ec405d4065f..e32a27ff6e3 100644
--- a/src/metabase/premium_features/core.clj
+++ b/src/metabase/premium_features/core.clj
@@ -31,7 +31,8 @@
plan-alias
quotas
TokenStatus
- clear-cache!]
+ clear-cache!
+ transform-stats]
(metabase.premium-features.settings
active-users-count
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index 1547c3bd821..e5dfa8ff6df 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -127,7 +127,7 @@
(defenterprise transform-stats
"Stats for Transforms"
- metabase.transforms.core
+ metabase-enterprise.transforms.core
[]
{:transform-native-runs 0
:transform-python-runs 0
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index e6b9a81a652..26ff5792cf6 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -1,13 +1,11 @@
(ns metabase.transforms.core
"API namespace for the `metabase.transforms` module."
(:require
- [java-time.api :as t]
[metabase.models.transforms.transform]
[metabase.models.transforms.transform-run]
[metabase.transforms.settings]
[metabase.transforms.util]
- [potemkin :as p]
- [toucan2.core :as t2]))
+ [potemkin :as p]))
(p/import-vars
[metabase.transforms.settings
@@ -25,25 +23,3 @@
timeout-run!]
[metabase.models.transforms.transform
update-transform-tags!])
-
-(defn transform-stats
- "Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.
- Rolling stats are included under :transform-rolling (up to date totals for today)"
- []
- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))
- yesterday-utc (t/minus today-utc (t/days 1))
- count-runs (fn [source-types date]
- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]
- :from [[:transform_run :r]]
- :join [[:transform :t] [:= :t.id :r.transform_id]]
- :where [:and
- [:= :r.status "succeeded"]
- [:in :t.source_type source-types]
- [:= [:cast :r.end_time :date] [:cast date :date]]]}))
- 0))]
- {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)
- :transform-python-runs (count-runs ["python"] yesterday-utc)
- :transform-usage-date (str (t/local-date yesterday-utc))
- :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)
- :transform-rolling-python-runs (count-runs ["python"] today-utc)
- :transform-rolling-usage-date (str (t/local-date today-utc))}))