The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Python transforms have dependencies (#68572)
Williama79917842005
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex 267f8a12170..0dfbc75fb3d 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -2,6 +2,7 @@(:require[metabase-enterprise.dependencies.native-validation :as deps.native][metabase-enterprise.dependencies.schema :as deps.schema]+ [metabase-enterprise.transforms.core :as transforms][metabase.documents.prose-mirror :as prose-mirror][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -43,19 +44,31 @@query-depsparam-card-ids)))+(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps+ "Given a Python Transform (in Toucan form), return its upstream dependencies."+ [transform]+ {:table (->> (get-in transform [:source :source-tables])+ vals+ (into #{}+ (keep #(if (map? %)+ (:table_id %)+ %))))})+(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps"Given a Transform (in Toucan form), return its upstream dependencies."- [{{:keys [query], source-type :type} :source :as transform} :-+ [{{:keys [query]} :source :as transform} :-[:map[:source [:multi {:dispatch (comp keyword :type)}[:query[:map [:query ::lib.schema/query]]][:python[:map]]]]]]- (if (= (keyword source-type) :query)- (upstream-deps:query query)- (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)- {})))+ (let [source-type (transforms/transform-type transform)]+ (case source-type+ :query (upstream-deps:query query)+ :python (upstream-deps:python-transform transform)+ (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)+ {}))))(mu/defn upstream-deps:snippet :- ::deps.schema/upstream-deps"Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.cljindex 0056f61c220..99561aa48f8 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj@@ -4,6 +4,7 @@[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types][metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.transforms.core :as transforms][metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.premium-features.core :as premium-features]@@ -89,8 +90,8 @@(derive :event/update-transform ::transform-deps);; On *saving* a transform, the upstream deps of its query are computed and saved.-(defn- drop-outdated-target-dep! [{:keys [id source target] :as _transform}]- (let [db-id (some-> source :query :database)+(defn- drop-outdated-target-dep! [{:keys [id target] :as transform}]+ (let [db-id (transforms/transform-source-database transform)downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency:from_entity_type :table:to_entity_type :transformdiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex f5931ea1a94..117023c2b19 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -14,7 +14,7 @@(def current-dependency-analysis-version"Current version of the dependency analysis logic.This should be incremented when the dependency analysis logic changes."- 4)+ 5)(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.cljindex 06c18cd0953..291c32de03d 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj@@ -15,7 +15,9 @@[metabase-enterprise.transforms.utilnative-query-transform?python-transform?- query-transform?]+ query-transform?+ transform-source-database+ transform-type][metabase-enterprise.transforms.models.transform-runtimeout-run!])diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.cljindex a03303ffcfc..2a01d851c22 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj@@ -5,7 +5,6 @@[metabase-enterprise.transforms.util :as transforms.util][metabase.driver :as driver][metabase.driver.util :as driver.u]- [metabase.events.core :as events][metabase.lib.schema.common :as schema.common][metabase.query-processor.compile :as qp.compile][metabase.util.log :as log]@@ -76,13 +75,10 @@(transforms.util/run-cancelable-transform!run-id driver transform-details(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]- (transforms.util/sync-target! target database)- ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.- (events/publish-event! :event/transform-run-complete {:object transform-details}))- ;; Creating an index after sync means the filter column is known in the appdb.- ;; The index would be synced the next time sync runs, but at time of writing, index sync is disabled.- (transforms.util/execute-secondary-index-ddl-if-required! transform run-id database target)))+ (transforms.util/handle-transform-complete!+ :run-id run-id+ :transform transform+ :db database)))(catch Throwable t(log/error t "Error executing transform")(when start-promisediff --git a/enterprise/backend/src/metabase_enterprise/transforms/schema.clj b/enterprise/backend/src/metabase_enterprise/transforms/schema.cljindex 8fc48dfc54c..197ee88aec7 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/schema.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/schema.clj@@ -75,8 +75,13 @@["table" ::table-target]["table-incremental" ::table-incremental-target]])+(mr/def ::id pos-int?)++(mr/def ::run-id pos-int?)+(mr/def ::transform[:map+ [:id ::id][:description {:optional true} [:maybe :string]][:name :string][:source [:ref ::transform-source]]diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex 068fdf7cd0a..e0e75e438fe 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -9,10 +9,12 @@[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation][metabase-enterprise.transforms.interface :as transforms.i][metabase-enterprise.transforms.models.transform-run :as transform-run]+ [metabase-enterprise.transforms.schema :as transforms.schema][metabase-enterprise.transforms.settings :as transforms.settings][metabase.driver :as driver][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.util :as driver.u]+ [metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -48,10 +50,15 @@(keyword schema name)(keyword name)))+(defn transform-type+ "Get the type of a transform"+ [transform]+ (-> transform :source :type keyword))+(defn query-transform?"Check if this is a query transform: native query / mbql query."[transform]- (= :query (-> transform :source :type keyword)))+ (= :query (transform-type transform)))(defn native-query-transform?"Check if this is a native query transform.@@ -64,7 +71,14 @@(defn python-transform?"Check if this is a Python transform."[transform]- (= :python (-> transform :source :type keyword)))+ (= :python (transform-type transform)))++(defn transform-source-database+ "Get the source database from a transform"+ [transform]+ (case (transform-type transform)+ :query (-> transform :source :query :database)+ :python (-> transform :source :source-database)))(defn normalize-transform"Normalize a transform's source query, similar to how transforms are normalized when read from the database.@@ -663,3 +677,29 @@(transforms.instrumentation/with-stage-timing [run-id [:import :create-incremental-filter-index]](log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))(driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))++(mu/defn handle-transform-complete!+ "Handles followup tasks for when a transform has completed.++ Specifically, this syncs the target db, publishes a `:event/transform-run-complete` event, and potentially updates+ the target table's index.++ See [[metabase.transforms-util/decide-secondary-index-ddl]] for details on the index handling."+ [& {:keys [run-id transform db]}+ :- [:map+ [:run-id ::transforms.schema/run-id]+ [:transform ::transforms.schema/transform]+ [:db [:fn {:error/message "Must a t2 database object"} #(= (t2/model %) :model/Database)]]]]+ (let [target (:target transform)]+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]+ (sync-target! target db)+ ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (:id db)+ :transform-id (:id transform)+ :transform-type (keyword (:type target))+ :output-schema (:schema target)+ :output-table (qualified-table-name (:engine db) target)}})+ ;; Creating an index after sync means the filter column is known in the appdb.+ ;; The index would be synced the next time sync runs, but at time of writing, index sync is disabled.+ (execute-secondary-index-ddl-if-required! transform run-id db target))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 6f6854b101b..a51f726228e 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -377,9 +377,10 @@ex-message-fn #(exceptional-run-message message-log %)result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]](transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]- (transforms.util/sync-target! target db))- (transforms.util/execute-secondary-index-ddl-if-required! transform run-id db target)+ (transforms.util/handle-transform-complete!+ :run-id run-id+ :transform transform+ :db db){:run_id run-id:result result}))(catch Throwable tdiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex 081d11a7bf7..6199a1471df 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -279,6 +279,164 @@(t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet :id snippet-id)))(is (empty? (t2/select :model/Dependency :from_entity_id snippet-id :from_entity_type :snippet)))))))))))+(deftest ^:sequential native-transform-updates-dependencies-test+ (testing "native transform update events trigger dependency calculations"+ (run-with-dependencies-setup+ (fn [mp]+ (let [source {:query (lib/native-query mp "select * from orders")+ :type :query}]+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})+ (is (= models.dependency/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (is (=? [{:from_entity_type :transform,+ :from_entity_id transform-id,+ :to_entity_type :table,+ :to_entity_id (mt/id :orders)}]+ (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))++(deftest ^:sequential mbql-transform-updates-dependencies-test+ (testing "mbql transform update events trigger dependency calculations"+ (run-with-dependencies-setup+ (fn [mp]+ (let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ :type :query}]+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})+ (is (= models.dependency/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (is (=? [{:from_entity_type :transform,+ :from_entity_id transform-id,+ :to_entity_type :table,+ :to_entity_id (mt/id :orders)}]+ (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))++(deftest ^:sequential python-transform-updates-dependencies-test+ (testing "python transform update events trigger dependency calculations"+ (run-with-dependencies-setup+ (fn [_]+ (let [source {:type :python,+ :source-database 2,+ :source-tables {"orders" {:database_id (mt/id),+ :schema "public",+ :table "orders",+ :table_id (mt/id :orders)}},+ :body+ "import pandas as pd\n\ndef transform(orders):\n return orders"}]+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})+ (is (= models.dependency/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (is (=? [{:from_entity_type :transform,+ :from_entity_id transform-id,+ :to_entity_type :table,+ :to_entity_id (mt/id :orders)}]+ (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))++(deftest ^:sequential transform-run-updates-dependencies-test+ (testing "transform run events trigger dependency calculations"+ (run-with-dependencies-setup+ (fn [_]+ (let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]+ (mt/with-temp [:model/Transform {transform-id :id} {:target target}+ :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}]+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (= models.dependency/current-dependency-analysis-version+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))++(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test+ (testing "python transform update events handles downstream dependencies"+ (run-with-dependencies-setup+ (fn [_]+ (let [source {:type :python,+ :source-database (mt/id),+ :source-tables {"orders" {:database_id (mt/id),+ :schema "public",+ :table "orders",+ :table_id (mt/id :orders)}},+ :body+ "import pandas as pd\n\ndef transform(orders):\n return orders"}+ target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target+ :source source}+ :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]+ (testing "initial run"+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "keeping target"+ (events/publish-event! :event/update-transform+ {:object transform+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "changing target update"+ (events/publish-event! :event/update-transform+ {:object (assoc-in transform [:target :name] "test_table2")+ :user-id api/*current-user-id*})+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))++(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test+ (testing "query transform update events handles downstream dependencies"+ (run-with-dependencies-setup+ (fn [mp]+ (let [source {:query (lib/native-query mp "select * from orders")+ :type :query}+ target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target :source source}+ :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]+ (testing "initial run"+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "Other"+ :output-table :test_table+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "keeping target"+ (events/publish-event! :event/update-transform+ {:object transform+ :user-id api/*current-user-id*})+ (is (=? [{:from_entity_type :table+ :from_entity_id table-id,+ :to_entity_type :transform,+ :to_entity_id transform-id}]+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))+ (testing "changing target update"+ (events/publish-event! :event/update-transform+ {:object (assoc-in transform [:target :name] "test_table2")+ :user-id api/*current-user-id*})+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))+(deftest transform-dependency-calculation-error-handling-test(testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"(run-with-dependencies-setup