The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Move transform metering back into EE module (#69840)
Dan Stoneacf4530cc6c3
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 1eaf8981e41..6923c670f38 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2435,7 +2435,7 @@enterprise/transforms{:team "Querying":api #{}- :uses #{}}+ :uses #{premium-features}}enterprise/transforms-python{:team "Querying"diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.cljindex 4522dddfb89..0cacb5bd971 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj@@ -1,9 +1,34 @@(ns metabase-enterprise.transforms.core- "Enterprise module placeholder for transforms.+ "Enterprise module for transforms used for both tests and metering.Transforms are weird: they have different behavior on OSS vs EE. We need to test this, so we need testsin `enterprise/backend/test/metabase_enterprise/transforms/api_test.clj`.- But we need a 'real' module, because all tests must belong to a module.+ But we need a 'real' module, because all tests must belong to a module."+ (:require+ [java-time.api :as t]+ [metabase.premium-features.core :refer [defenterprise]]+ [toucan2.core :as t2]))- Hence, this placeholder.")+(defenterprise transform-stats+ "Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.+ Rolling stats are included under :transform-rolling (up to date totals for today)"+ :feature :none+ []+ (let [today-utc (t/offset-date-time (t/zone-offset "+00"))+ yesterday-utc (t/minus today-utc (t/days 1))+ count-runs (fn [source-types date]+ (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]+ :from [[:transform_run :r]]+ :join [[:transform :t] [:= :t.id :r.transform_id]]+ :where [:and+ [:= :r.status "succeeded"]+ [:in :t.source_type source-types]+ [:= [:cast :r.end_time :date] [:cast date :date]]]}))+ 0))]+ {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)+ :transform-python-runs (count-runs ["python"] yesterday-utc)+ :transform-usage-date (str (t/local-date yesterday-utc))+ :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)+ :transform-rolling-python-runs (count-runs ["python"] today-utc)+ :transform-rolling-usage-date (str (t/local-date today-utc))}))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/core_test.cljnew file mode 100644index 00000000000..201f1f0d065--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms/core_test.clj@@ -0,0 +1,63 @@+(ns metabase-enterprise.transforms.core-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.models.transforms.transform-run :as transform-run]+ [metabase.premium-features.core :as premium-features]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db))++(deftest transform-stats-rolling-to-yesterday-test+ (testing "completed runs appear in rolling stats, then move to yesterday's stats when end_time is backdated"+ (mt/with-temp [:model/Transform {native-id :id} {}+ :model/Transform {python-id :id} {:source {:type "python"+ :source-database (mt/id)}+ :target {:type "table"+ :name (str "test_python_" (random-uuid))+ :database (mt/id)}}]+ (let [stats-before (premium-features/transform-stats)+ native-before (:transform-native-runs stats-before)+ rolling-native-before (:transform-rolling-native-runs stats-before)+ python-before (:transform-python-runs stats-before)+ rolling-python-before (:transform-rolling-python-runs stats-before)+ yesterday-utc (t/minus (t/offset-date-time (t/zone-offset "+00")) (t/days 1))]++ (testing "native"+ (let [{run-id :id} (transform-run/start-run! native-id {:run_method "manual"})]+ (transform-run/succeed-started-run! run-id)++ (testing "completed run appears in rolling stats only"+ (let [stats (premium-features/transform-stats)]+ (is (= (inc rolling-native-before) (:transform-rolling-native-runs stats)))+ (is (= native-before (:transform-native-runs stats)))))++ (testing "a run for yesterday is visible under yesterday's stats only"+ (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})+ (let [stats (premium-features/transform-stats)]+ (is (= rolling-native-before (:transform-rolling-native-runs stats)))+ (is (= (inc native-before) (:transform-native-runs stats)))))))++ (testing "python stats not modified by native runs"+ (let [stats (premium-features/transform-stats)]+ (is (= python-before (:transform-python-runs stats)))+ (is (= rolling-python-before (:transform-rolling-python-runs stats)))))++ (testing "python"+ (let [{run-id :id} (transform-run/start-run! python-id {:run_method "manual"})]+ (transform-run/succeed-started-run! run-id)++ (testing "completed run appears in rolling stats only"+ (let [stats (premium-features/transform-stats)]+ (is (= (inc rolling-python-before) (:transform-rolling-python-runs stats)))+ (is (= python-before (:transform-python-runs stats)))))++ (testing "a run for yesterday is visible under yesterday's state only"+ (t2/update! :model/TransformRun :id run-id {:end_time yesterday-utc})+ (let [stats (premium-features/transform-stats)]+ (is (= rolling-python-before (:transform-rolling-python-runs stats)))+ (is (= (inc python-before) (:transform-python-runs stats)))))))))))diff --git a/src/metabase/premium_features/core.clj b/src/metabase/premium_features/core.cljindex ec405d4065f..e32a27ff6e3 100644--- a/src/metabase/premium_features/core.clj+++ b/src/metabase/premium_features/core.clj@@ -31,7 +31,8 @@plan-aliasquotasTokenStatus- clear-cache!]+ clear-cache!+ transform-stats](metabase.premium-features.settingsactive-users-countdiff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex 1547c3bd821..e5dfa8ff6df 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -127,7 +127,7 @@(defenterprise transform-stats"Stats for Transforms"- metabase.transforms.core+ metabase-enterprise.transforms.core[]{:transform-native-runs 0:transform-python-runs 0diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex e6b9a81a652..26ff5792cf6 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -1,13 +1,11 @@(ns metabase.transforms.core"API namespace for the `metabase.transforms` module."(:require- [java-time.api :as t][metabase.models.transforms.transform][metabase.models.transforms.transform-run][metabase.transforms.settings][metabase.transforms.util]- [potemkin :as p]- [toucan2.core :as t2]))+ [potemkin :as p]))(p/import-vars[metabase.transforms.settings@@ -25,25 +23,3 @@timeout-run!][metabase.models.transforms.transformupdate-transform-tags!])--(defn transform-stats- "Calculate successful transform runs over a window of the previous UTC day 00:00-23:59.- Rolling stats are included under :transform-rolling (up to date totals for today)"- []- (let [today-utc (t/offset-date-time (t/zone-offset "+00"))- yesterday-utc (t/minus today-utc (t/days 1))- count-runs (fn [source-types date]- (or (:cnt (t2/query-one {:select [[[:count :r.id] :cnt]]- :from [[:transform_run :r]]- :join [[:transform :t] [:= :t.id :r.transform_id]]- :where [:and- [:= :r.status "succeeded"]- [:in :t.source_type source-types]- [:= [:cast :r.end_time :date] [:cast date :date]]]}))- 0))]- {:transform-native-runs (count-runs ["native" "mbql"] yesterday-utc)- :transform-python-runs (count-runs ["python"] yesterday-utc)- :transform-usage-date (str (t/local-date yesterday-utc))- :transform-rolling-native-runs (count-runs ["native" "mbql"] today-utc)- :transform-rolling-python-runs (count-runs ["python"] today-utc)- :transform-rolling-usage-date (str (t/local-date today-utc))}))