Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Python transforms have dependencies (#68572)

Williama79917842005
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
index 267f8a12170..0dfbc75fb3d 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -2,6 +2,7 @@
(:require
[metabase-enterprise.dependencies.native-validation :as deps.native]
[metabase-enterprise.dependencies.schema :as deps.schema]
+ [metabase-enterprise.transforms.core :as transforms]
[metabase.documents.prose-mirror :as prose-mirror]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -43,19 +44,31 @@
query-deps
param-card-ids)))
+(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
+ "Given a Python Transform (in Toucan form), return its upstream dependencies."
+ [transform]
+ {:table (->> (get-in transform [:source :source-tables])
+ vals
+ (into #{}
+ (keep #(if (map? %)
+ (:table_id %)
+ %))))})
+
(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
"Given a Transform (in Toucan form), return its upstream dependencies."
- [{{:keys [query], source-type :type} :source :as transform} :-
+ [{{:keys [query]} :source :as transform} :-
[:map
[:source [:multi {:dispatch (comp keyword :type)}
[:query
[:map [:query ::lib.schema/query]]]
[:python
[:map]]]]]]
- (if (= (keyword source-type) :query)
- (upstream-deps:query query)
- (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
- {})))
+ (let [source-type (transforms/transform-type transform)]
+ (case source-type
+ :query (upstream-deps:query query)
+ :python (upstream-deps:python-transform transform)
+ (do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
+ {}))))
(mu/defn upstream-deps:snippet :- ::deps.schema/upstream-deps
"Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
index 0056f61c220..99561aa48f8 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
@@ -4,6 +4,7 @@
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
[metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.transforms.core :as transforms]
[metabase.events.core :as events]
[metabase.lib-be.core :as lib-be]
[metabase.premium-features.core :as premium-features]
@@ -89,8 +90,8 @@
(derive :event/update-transform ::transform-deps)
;; On *saving* a transform, the upstream deps of its query are computed and saved.
-(defn- drop-outdated-target-dep! [{:keys [id source target] :as _transform}]
- (let [db-id (some-> source :query :database)
+(defn- drop-outdated-target-dep! [{:keys [id target] :as transform}]
+ (let [db-id (transforms/transform-source-database transform)
downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency
:from_entity_type :table
:to_entity_type :transform
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index f5931ea1a94..117023c2b19 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -14,7 +14,7 @@
(def current-dependency-analysis-version
"Current version of the dependency analysis logic.
This should be incremented when the dependency analysis logic changes."
- 4)
+ 5)
(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/core.clj b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
index 06c18cd0953..291c32de03d 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/core.clj
@@ -15,7 +15,9 @@
[metabase-enterprise.transforms.util
native-query-transform?
python-transform?
- query-transform?]
+ query-transform?
+ transform-source-database
+ transform-type]
[metabase-enterprise.transforms.models.transform-run
timeout-run!])
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
index a03303ffcfc..2a01d851c22 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/query_impl.clj
@@ -5,7 +5,6 @@
[metabase-enterprise.transforms.util :as transforms.util]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
- [metabase.events.core :as events]
[metabase.lib.schema.common :as schema.common]
[metabase.query-processor.compile :as qp.compile]
[metabase.util.log :as log]
@@ -76,13 +75,10 @@
(transforms.util/run-cancelable-transform!
run-id driver transform-details
(fn [_cancel-chan] (driver/run-transform! driver transform-details opts))))
- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
- (transforms.util/sync-target! target database)
- ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
- (events/publish-event! :event/transform-run-complete {:object transform-details}))
- ;; Creating an index after sync means the filter column is known in the appdb.
- ;; The index would be synced the next time sync runs, but at time of writing, index sync is disabled.
- (transforms.util/execute-secondary-index-ddl-if-required! transform run-id database target)))
+ (transforms.util/handle-transform-complete!
+ :run-id run-id
+ :transform transform
+ :db database)))
(catch Throwable t
(log/error t "Error executing transform")
(when start-promise
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/schema.clj b/enterprise/backend/src/metabase_enterprise/transforms/schema.clj
index 8fc48dfc54c..197ee88aec7 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/schema.clj
@@ -75,8 +75,13 @@
["table" ::table-target]
["table-incremental" ::table-incremental-target]])
+(mr/def ::id pos-int?)
+
+(mr/def ::run-id pos-int?)
+
(mr/def ::transform
[:map
+ [:id ::id]
[:description {:optional true} [:maybe :string]]
[:name :string]
[:source [:ref ::transform-source]]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
index 068fdf7cd0a..e0e75e438fe 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj
@@ -9,10 +9,12 @@
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.interface :as transforms.i]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
+ [metabase-enterprise.transforms.schema :as transforms.schema]
[metabase-enterprise.transforms.settings :as transforms.settings]
[metabase.driver :as driver]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.util :as driver.u]
+ [metabase.events.core :as events]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -48,10 +50,15 @@
(keyword schema name)
(keyword name)))
+(defn transform-type
+ "Get the type of a transform"
+ [transform]
+ (-> transform :source :type keyword))
+
(defn query-transform?
"Check if this is a query transform: native query / mbql query."
[transform]
- (= :query (-> transform :source :type keyword)))
+ (= :query (transform-type transform)))
(defn native-query-transform?
"Check if this is a native query transform.
@@ -64,7 +71,14 @@
(defn python-transform?
"Check if this is a Python transform."
[transform]
- (= :python (-> transform :source :type keyword)))
+ (= :python (transform-type transform)))
+
+(defn transform-source-database
+ "Get the source database from a transform"
+ [transform]
+ (case (transform-type transform)
+ :query (-> transform :source :query :database)
+ :python (-> transform :source :source-database)))
(defn normalize-transform
"Normalize a transform's source query, similar to how transforms are normalized when read from the database.
@@ -663,3 +677,29 @@
(transforms.instrumentation/with-stage-timing [run-id [:import :create-incremental-filter-index]]
(log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))
(driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))
+
+(mu/defn handle-transform-complete!
+ "Handles followup tasks for when a transform has completed.
+
+ Specifically, this syncs the target db, publishes a `:event/transform-run-complete` event, and potentially updates
+ the target table's index.
+
+ See [[metabase.transforms-util/decide-secondary-index-ddl]] for details on the index handling."
+ [& {:keys [run-id transform db]}
+ :- [:map
+ [:run-id ::transforms.schema/run-id]
+ [:transform ::transforms.schema/transform]
+ [:db [:fn {:error/message "Must a t2 database object"} #(= (t2/model %) :model/Database)]]]]
+ (let [target (:target transform)]
+ (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
+ (sync-target! target db)
+ ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (:id db)
+ :transform-id (:id transform)
+ :transform-type (keyword (:type target))
+ :output-schema (:schema target)
+ :output-table (qualified-table-name (:engine db) target)}})
+ ;; Creating an index after sync means the filter column is known in the appdb.
+ ;; The index would be synced the next time sync runs, but at time of writing, index sync is disabled.
+ (execute-secondary-index-ddl-if-required! transform run-id db target))))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 6f6854b101b..a51f726228e 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -377,9 +377,10 @@
ex-message-fn #(exceptional-run-message message-log %)
result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
(transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
- (transforms.util/sync-target! target db))
- (transforms.util/execute-secondary-index-ddl-if-required! transform run-id db target)
+ (transforms.util/handle-transform-complete!
+ :run-id run-id
+ :transform transform
+ :db db)
{:run_id run-id
:result result}))
(catch Throwable t
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index 081d11a7bf7..6199a1471df 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -279,6 +279,164 @@
(t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet :id snippet-id)))
(is (empty? (t2/select :model/Dependency :from_entity_id snippet-id :from_entity_type :snippet)))))))))))
+(deftest ^:sequential native-transform-updates-dependencies-test
+ (testing "native transform update events trigger dependency calculations"
+ (run-with-dependencies-setup
+ (fn [mp]
+ (let [source {:query (lib/native-query mp "select * from orders")
+ :type :query}]
+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
+ (is (= models.dependency/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (is (=? [{:from_entity_type :transform,
+ :from_entity_id transform-id,
+ :to_entity_type :table,
+ :to_entity_id (mt/id :orders)}]
+ (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))
+
+(deftest ^:sequential mbql-transform-updates-dependencies-test
+ (testing "mbql transform update events trigger dependency calculations"
+ (run-with-dependencies-setup
+ (fn [mp]
+ (let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
+ :type :query}]
+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
+ (is (= models.dependency/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (is (=? [{:from_entity_type :transform,
+ :from_entity_id transform-id,
+ :to_entity_type :table,
+ :to_entity_id (mt/id :orders)}]
+ (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))
+
+(deftest ^:sequential python-transform-updates-dependencies-test
+ (testing "python transform update events trigger dependency calculations"
+ (run-with-dependencies-setup
+ (fn [_]
+ (let [source {:type :python,
+ :source-database 2,
+ :source-tables {"orders" {:database_id (mt/id),
+ :schema "public",
+ :table "orders",
+ :table_id (mt/id :orders)}},
+ :body
+ "import pandas as pd\n\ndef transform(orders):\n return orders"}]
+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
+ (is (= models.dependency/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (is (=? [{:from_entity_type :transform,
+ :from_entity_id transform-id,
+ :to_entity_type :table,
+ :to_entity_id (mt/id :orders)}]
+ (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))
+
+(deftest ^:sequential transform-run-updates-dependencies-test
+ (testing "transform run events trigger dependency calculations"
+ (run-with-dependencies-setup
+ (fn [_]
+ (let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
+ (mt/with-temp [:model/Transform {transform-id :id} {:target target}
+ :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}]
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (= models.dependency/current-dependency-analysis-version
+ (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))
+
+(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test
+ (testing "python transform update events handles downstream dependencies"
+ (run-with-dependencies-setup
+ (fn [_]
+ (let [source {:type :python,
+ :source-database (mt/id),
+ :source-tables {"orders" {:database_id (mt/id),
+ :schema "public",
+ :table "orders",
+ :table_id (mt/id :orders)}},
+ :body
+ "import pandas as pd\n\ndef transform(orders):\n return orders"}
+ target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target
+ :source source}
+ :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]
+ (testing "initial run"
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "keeping target"
+ (events/publish-event! :event/update-transform
+ {:object transform
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "changing target update"
+ (events/publish-event! :event/update-transform
+ {:object (assoc-in transform [:target :name] "test_table2")
+ :user-id api/*current-user-id*})
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
+
+(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test
+ (testing "query transform update events handles downstream dependencies"
+ (run-with-dependencies-setup
+ (fn [mp]
+ (let [source {:query (lib/native-query mp "select * from orders")
+ :type :query}
+ target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:target target :source source}
+ :model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
+ :model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}]
+ (testing "initial run"
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "Other"
+ :output-table :test_table
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "keeping target"
+ (events/publish-event! :event/update-transform
+ {:object transform
+ :user-id api/*current-user-id*})
+ (is (=? [{:from_entity_type :table
+ :from_entity_id table-id,
+ :to_entity_type :transform,
+ :to_entity_id transform-id}]
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
+ (testing "changing target update"
+ (events/publish-event! :event/update-transform
+ {:object (assoc-in transform [:target :name] "test_table2")
+ :user-id api/*current-user-id*})
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
+
(deftest transform-dependency-calculation-error-handling-test
(testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"
(run-with-dependencies-setup