Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Extract non-appdb module out of transforms (attempt #3) (#70245)

Chris Truter57586f242722
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 121ab6d0dec..08727d8db1b 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -422,6 +422,8 @@
ring.util.codec codec
ring.util.response response
saml20-clj.core saml
+ metabase.transforms-base.util transforms-base.u
+ metabase.transforms.util transforms.u
toucan2.core t2}}}
:lint-as
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index e320b1048d7..bd37fbb390b 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -870,6 +870,7 @@
metabase.models.interface
metabase.models.resolution
metabase.models.serialization
+ metabase.models.transforms.transform
metabase.models.util.spec-update
metabase.models.visualization-settings}
:uses #{api
@@ -886,6 +887,7 @@
search
settings
transforms
+ transforms-base
util}
:friends #{transforms}}
@@ -1574,7 +1576,6 @@
:api #{metabase.transforms.core
metabase.transforms.feature-gating
metabase.transforms.init
- metabase.transforms.interface
metabase.transforms.schema
metabase.transforms.util}
:uses #{analytics
@@ -1584,21 +1585,36 @@
driver
events
lib
- lib-be
models
premium-features
queries
query-processor
revisions
settings
- sync
task
task-history
+ transforms-base
util}
:friends #{models
enterprise/transforms-python
enterprise/workspaces}}
+ transforms-base
+ {:team "Querying"
+ :api #{metabase.transforms-base.init
+ metabase.transforms-base.interface
+ metabase.transforms-base.util}
+ :uses #{driver
+ events
+ lib
+ lib-be
+ query-processor
+ sync
+ util}
+ :friends #{transforms
+ enterprise/transforms-python
+ enterprise/workspaces}}
+
transforms-inspector
{:team "Gadget"
:api #{metabase.transforms-inspector.core
@@ -1608,7 +1624,7 @@
lib-be
query-processor
sql-tools
- transforms
+ transforms-base
util}}
transforms-rest
@@ -1625,6 +1641,7 @@
server
sql-tools
transforms
+ transforms-base
transforms-inspector
util}}
@@ -2136,6 +2153,7 @@
settings
task
transforms
+ transforms-base
util}}
enterprise/email
@@ -2253,6 +2271,7 @@
system
task
transforms
+ transforms-base
users
util
warehouses
@@ -2476,6 +2495,7 @@
query-processor
settings
transforms
+ transforms-base
util}}
enterprise/upload-management
@@ -2509,6 +2529,7 @@
sql-tools
system
transforms
+ transforms-base
util
enterprise/transforms-python}}}
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
index e89f8e6d18e..21511052820 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -7,7 +7,7 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
[metabase.queries.schema :as queries.schema]
- [metabase.transforms.core :as transforms]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
@@ -60,7 +60,7 @@
[:python
;; If the upstream table doesn't exist yet, table_id will be nil
[:map [:source-tables {:optional true} [:map-of :string [:or :int [:map [:table_id [:maybe :int]]]]]]]]]]]]
- (let [source-type (transforms/transform-type transform)]
+ (let [source-type (transforms-base.u/transform-type transform)]
(case source-type
:query (upstream-deps:query query)
:python (upstream-deps:python-transform transform)
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
index d0be3f46e2e..c1518645217 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj
@@ -8,7 +8,7 @@
[metabase.config.core :as config]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util.json :as json]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -146,9 +146,9 @@
(try
(if (and (= (:type item) "transform")
(not (:source_type item)))
- (let [transform (transforms.util/normalize-transform item)]
+ (let [transform (transforms-base.u/normalize-transform item)]
(assoc transform
- :source_type (transforms.util/transform-source-type (:source transform))))
+ :source_type (transforms-base.u/transform-source-type (:source transform))))
item)
(catch Exception e
(log/error e "Error annotating transform source type for metabot context")
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
new file mode 100644
index 00000000000..d0eda8d5e5f
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -0,0 +1,376 @@
+(ns metabase-enterprise.transforms-python.base
+ "Base Python transform execution - core logic without transform_run tracking.
+
+ This namespace handles Python transform execution and returns results in memory
+ rather than writing to transform_run rows. Logs are returned in the result map
+ instead of being saved to transform_run.message."
+ (:require
+ [clojure.core.async :as a]
+ [clojure.java.io :as io]
+ [clojure.string :as str]
+ [metabase-enterprise.transforms-python.python-runner :as python-runner]
+ [metabase-enterprise.transforms-python.s3 :as s3]
+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
+ [metabase.driver :as driver]
+ [metabase.driver.util :as driver.u]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.instrumentation :as transforms.instrumentation]
+ [metabase.util :as u]
+ [metabase.util.format :as u.format]
+ [metabase.util.i18n :as i18n]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2])
+ (:import
+ (java.io File)
+ (java.nio.file Files)
+ (java.nio.file.attribute FileAttribute)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------- Interface Implementations -------------------------------------------------
+
+(defmethod transforms-base.i/target-db-id :python
+ [transform]
+ (-> transform :target :database))
+
+(defmethod transforms-base.i/source-db-id :python
+ [transform]
+ (-> transform :source :source-database))
+
+(defn- source-table-value->dependency
+ "Convert a source table value (int or ref map) to a dependency map."
+ [v]
+ (cond
+ ;; Integer table ID - direct table dependency
+ (int? v)
+ {:table v}
+
+ ;; Map with resolved table_id - direct table dependency
+ (:table_id v)
+ {:table (:table_id v)}
+
+ ;; Map without table_id - return ref for ordering system to resolve
+ :else
+ {:table-ref (select-keys v [:database_id :schema :table])}))
+
+(defmethod transforms-base.i/table-dependencies :python
+ [transform]
+ (into #{}
+ (map source-table-value->dependency)
+ (vals (get-in transform [:source :source-tables]))))
+
+;;; ------------------------------------------------- Message Log (in-memory only) -------------------------------------------------
+
+(defn empty-message-log
+ "Returns a new message log for accumulating execution logs in memory."
+ []
+ (atom {:pre-python []
+ :python nil
+ :post-python []}))
+
+(defn- log!
+ "Appends a string to the message log."
+ [message-log s]
+ (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
+ nil)
+
+(defn- replace-python-logs!
+ "Called when events output is received to copy those events into the message-log."
+ [message-log events]
+ (swap! message-log assoc :python events)
+ nil)
+
+(defn message-log->string
+ "Presents the content of the log as a string."
+ [message-log]
+ (let [{:keys [pre-python python post-python]} @message-log]
+ (str/join "\n" (concat pre-python
+ (map :message python)
+ post-python))))
+
+;;; ------------------------------------------------- Transfer Strategies -------------------------------------------------
+
+(defn- table-schema [table-name metadata]
+ {:name (if (keyword? table-name) table-name (keyword table-name))
+ :columns (mapv (fn [{:keys [name base_type]}]
+ {:name name
+ :type (python-runner/restricted-insert-type base_type)
+ :nullable? true})
+ (:fields metadata))})
+
+(defn- insert-data!
+ "Insert data from source into an existing table."
+ [driver db-id table-schema data-source]
+ (let [data-source (assoc data-source :table-schema table-schema)]
+ (driver/insert-from-source! driver db-id table-schema data-source)))
+
+(defn- create-table-and-insert-data!
+ "Create a table from metadata and insert data from source."
+ [driver db-id table-schema data-source]
+ (transforms-base.u/create-table-from-schema! driver db-id table-schema)
+ (insert-data! driver db-id table-schema data-source))
+
+(defn- transfer-with-rename-tables-strategy!
+ "Transfer data using the rename-tables*! multimethod with atomicity guarantees."
+ [driver db-id table-name metadata data-source]
+ (let [source-table-name (transforms-base.u/temp-table-name driver (namespace table-name))
+ temp-table-name (u/poll {:thunk #(transforms-base.u/temp-table-name driver (namespace table-name))
+ :done? #(not= source-table-name %)
+ :interval-ms 1})]
+ (log/info "Using rename-tables strategy with atomicity guarantees")
+ (try
+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
+ (transforms-base.u/rename-tables! driver db-id {table-name temp-table-name
+ source-table-name table-name})
+ (transforms-base.u/drop-table! driver db-id temp-table-name)
+ (catch Exception e
+ (log/error e "Failed to transfer data using rename-tables strategy")
+ (try
+ (transforms-base.u/drop-table! driver db-id source-table-name)
+ (catch Exception _))
+ (throw e)))))
+
+(defn- transfer-with-create-drop-rename-strategy!
+ "Transfer data using create + drop + rename to minimize time without data."
+ [driver db-id table-name metadata data-source]
+ (let [source-table-name (transforms-base.u/temp-table-name driver (namespace table-name))]
+ (log/info "Using create-drop-rename strategy to minimize downtime")
+ (try
+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
+ (transforms-base.u/drop-table! driver db-id table-name)
+ (driver/rename-table! driver db-id source-table-name table-name)
+ (catch Exception e
+ (log/error e "Failed to transfer data using create-drop-rename strategy")
+ (try
+ (transforms-base.u/drop-table! driver db-id source-table-name)
+ (catch Exception _))
+ (throw e)))))
+
+(defn- transfer-with-drop-create-fallback-strategy!
+ "Transfer data using drop + create fallback strategy."
+ [driver db-id table-name metadata data-source]
+ (log/info "Using drop-create fallback strategy")
+ (try
+ (transforms-base.u/drop-table! driver db-id table-name)
+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
+ (catch Exception e
+ (log/error e "Failed to transfer data using drop-create fallback strategy")
+ (throw e))))
+
+(defmulti ^:private transfer-file-to-db
+ {:arglists '([driver db transform metadata temp-file])}
+ (fn [_ _ transform _ _] (-> transform :target :type keyword)))
+
+(defmethod transfer-file-to-db :table-incremental
+ [driver {db-id :id}
+ {:keys [target] :as transform}
+ metadata temp-file]
+ (let [table-name (transforms-base.u/qualified-table-name driver target)
+ table-exists? (transforms-base.u/target-table-exists? transform)
+ data-source {:type :jsonl-file
+ :file temp-file}]
+ (if (not table-exists?)
+ (do
+ (log/info "New table")
+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
+ (insert-data! driver db-id (table-schema table-name metadata) data-source))))
+
+(defmethod transfer-file-to-db :table
+ [driver {db-id :id :as db}
+ {:keys [target] :as transform}
+ metadata temp-file]
+ (let [table-name (transforms-base.u/qualified-table-name driver target)
+ table-exists? (transforms-base.u/target-table-exists? transform)
+ data-source {:type :jsonl-file
+ :file temp-file}]
+ (cond
+ (not table-exists?)
+ (do
+ (log/info "New table")
+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
+
+ (driver.u/supports? driver :atomic-renames db)
+ (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
+
+ (driver.u/supports? driver :rename db)
+ (transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
+
+ :else
+ (transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
+
+;;; ------------------------------------------------- Cancellation -------------------------------------------------
+
+(defn- start-cancellation-process!
+ "Starts a core.async process that sends a cancellation request to the python executor if cancel-chan receives a value."
+ [server-url run-id cancel-chan]
+ (a/go (when (a/<! cancel-chan)
+ (python-runner/cancel-python-code-http-call! server-url run-id))))
+
+;;; ------------------------------------------------- Core Execution -------------------------------------------------
+
+(defn- run-python-transform-impl!
+ "Core Python transform execution. Returns {:status :result :logs :events}.
+
+ Options:
+ - `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"
+ [{:keys [source] :as transform} db run-id cancel-chan message-log {:keys [with-stage-timing-fn]}]
+ ;; Resolve name-based source table refs to table IDs (throws if any not found)
+ (let [resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables source))]
+ (with-open [shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
+ (let [driver (:engine db)
+ server-url (transforms-python.settings/python-runner-url)
+ _ (python-runner/copy-tables-to-s3! {:run-id run-id
+ :shared-storage @shared-storage-ref
+ :source (assoc source :source-tables resolved-source-tables)
+ :cancel-chan cancel-chan
+ :limit (:limit source)
+ :transform-id (:id transform)})
+ _ (start-cancellation-process! server-url run-id cancel-chan)
+ {:keys [status body] :as response}
+ (python-runner/execute-python-code-http-call!
+ {:server-url server-url
+ :code (:body source)
+ :run-id run-id
+ :table-name->id resolved-source-tables
+ :shared-storage @shared-storage-ref})
+
+ output-manifest (python-runner/read-output-manifest @shared-storage-ref)
+ events (python-runner/read-events @shared-storage-ref)]
+ (replace-python-logs! message-log events)
+ (if (not= 200 status)
+ (throw (ex-info "Python runner call failed"
+ {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
+ :status-code 400
+ :api-status-code status
+ :body body
+ :events events
+ :timeout (:timeout body)}))
+ (try
+ (let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
+ temp-file ^File (.toFile temp-path)]
+ (when-not (seq (:fields output-manifest))
+ (throw (ex-info "No fields in metadata"
+ {:metadata output-manifest
+ :raw-body body
+ :events events})))
+ (try
+ (with-open [in (python-runner/open-output @shared-storage-ref)]
+ (io/copy in temp-file))
+ ;; Transfer file to database with instrumentation
+ (let [file-size (.length temp-file)
+ do-transfer (fn [] (transfer-file-to-db driver db transform output-manifest temp-file))]
+ (if with-stage-timing-fn
+ (with-stage-timing-fn run-id [:import :file-to-dwh] do-transfer)
+ (do-transfer))
+ (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+ (finally
+ (.delete temp-file))))
+ response
+ (catch Exception e
+ (log/error e "Failed to create resulting table")
+ (throw (ex-info "Failed to create the resulting table"
+ {:transform-message (or (:transform-message (ex-data e))
+ (i18n/tru "Failed to create the resulting table"))}
+ e)))))))))
+
+;;; ------------------------------------------------- Base Execution Entry Point -------------------------------------------------
+
+(defn run-python-transform!
+ "Execute Python transform. Returns result map.
+
+ Does:
+ - Resolve source tables
+ - Copy to S3
+ - Call python runner
+ - Transfer result to DB
+ - Sync target
+
+ Does NOT:
+ - Create transform_run row
+ - Poll/save logs to transform_run.message
+
+ Options:
+ - `:cancelled?` - (fn [] boolean), polled to check for cancellation
+ - `:run-id` - optional, for cancellation signaling and instrumentation
+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk] result)
+ - `:message-log` - optional, pre-created message log atom (for scheduled execution log polling)
+ - `:cancel-chan` - optional, pre-created promise-chan for cancellation (avoids bridging latency)
+
+ Returns:
+ {:status :succeeded | :failed | :cancelled
+ :result <http response>
+ :logs <string>
+ :error <exception if failed>}"
+ [transform {:keys [cancelled? run-id with-stage-timing-fn message-log cancel-chan]}]
+ (assert (transforms-base.u/python-transform? transform) "Transform must be a python transform")
+ (let [message-log (or message-log (empty-message-log))]
+ (try
+ ;; Check cancellation before starting
+ (when (and cancelled? (cancelled?))
+ (throw (ex-info "Transform cancelled before start" {:status :cancelled})))
+
+ (let [{:keys [target] transform-id :id} transform
+ db (t2/select-one :model/Database (:database target))
+ ;; Use run-id if provided, otherwise generate a temp one for python runner
+ effective-run-id (or run-id (rand-int Integer/MAX_VALUE))
+ cancel-chan (or cancel-chan
+ (let [ch (a/promise-chan)]
+ ;; Bridge cancelled? to cancel-chan
+ (when cancelled?
+ (a/go
+ (loop []
+ (when-not (a/poll! ch)
+ (if (cancelled?)
+ (a/>! ch :cancel!)
+ (do
+ (a/<! (a/timeout 100))
+ (recur)))))))
+ ch))
+ start-ms (u/start-timer)]
+
+ (log! message-log (i18n/tru "Executing Python transform"))
+ (log/info "Executing Python transform" transform-id "with target" (pr-str target))
+
+ (let [result (run-python-transform-impl! transform db effective-run-id cancel-chan message-log
+ {:with-stage-timing-fn with-stage-timing-fn})]
+ (log! message-log (i18n/tru "Python execution finished successfully in {0}"
+ (u.format/format-milliseconds (u/since-ms start-ms))))
+
+ ;; Check cancellation after python
+ (when (and cancelled? (cancelled?))
+ (throw (ex-info "Transform cancelled after python execution" {:status :cancelled})))
+
+ {:status :succeeded
+ :result result
+ :logs (message-log->string message-log)}))
+
+ (catch Exception e
+ (let [data (ex-data e)
+ logs (message-log->string message-log)
+ error-message (or (:transform-message data) (ex-message e))]
+ (cond
+ (= :cancelled (:status data))
+ {:status :cancelled
+ :error e
+ :logs logs}
+
+ (:timeout data)
+ {:status :timeout
+ :error e
+ :logs (or (when-let [events (:events data)]
+ (str/join "\n" (map :message events)))
+ logs)}
+
+ :else
+ (do
+ (log/error e "Error executing Python transform")
+ {:status :failed
+ :error e
+ :logs (str logs "\n" error-message)})))))))
+
+;;; ------------------------------------------------- Interface Implementation -------------------------------------------------
+
+(defmethod transforms-base.i/execute-base! :python
+ [transform opts]
+ (run-python-transform! transform opts))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 30df0fa472b..1a88ad71a40 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -1,21 +1,16 @@
(ns metabase-enterprise.transforms-python.execute
(:require
[clojure.core.async :as a]
- [clojure.java.io :as io]
[clojure.string :as str]
+ [metabase-enterprise.transforms-python.base :as base]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
- [metabase-enterprise.transforms-python.s3 :as s3]
- [metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.connection :as driver.conn]
- [metabase.driver.util :as driver.u]
- [metabase.transforms.core :as transforms]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.instrumentation :as transforms.instrumentation]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.util :as transforms.util]
- [metabase.util :as u]
- [metabase.util.format :as u.format]
+ [metabase.transforms.util :as transforms.u]
[metabase.util.i18n :as i18n]
[metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
@@ -23,52 +18,16 @@
(:import
(java.io Closeable)
(java.net SocketException)
- (java.nio.file Files)
- (java.nio.file.attribute FileAttribute)
(java.time Duration)))
(set! *warn-on-reflection* true)
-(defn- empty-message-log
- "Returns a new message log.
-
- Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.
- The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.
-
- Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.
-
- The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."
- []
- (atom {:pre-python [] ; log! outputs previous to the python execution, i.e table read progress
- :python nil ; events json structured logs from the /logs endpoint
- :post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target
-
-(defn- log!
- "Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."
- [message-log s]
- (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
- nil)
-
-(defn- replace-python-logs!
- "Called when events output is received from a /logs call to copy those events into the message-log."
- [message-log events]
- (swap! message-log assoc :python events)
- nil)
-
-(defn- message-log->transform-run-message
- "Presents the content of the log as a string suitable for storage in transform_run.message."
- [message-log]
- (let [{:keys [pre-python python post-python]} @message-log]
- (str/join "\n" (concat pre-python
- (map :message python)
- post-python))))
-
(defn- save-log-to-transform-run-message!
"Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."
[run-id message-log]
(t2/update! :model/TransformRun
:id run-id
- {:message (message-log->transform-run-message message-log)}))
+ {:message (base/message-log->string message-log)}))
(def ^:private ^Duration python-message-loop-sleep-duration
(Duration/ofMillis 1000))
@@ -93,7 +52,7 @@
(do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
(recur))
(do
- (replace-python-logs! message-log events)
+ (swap! message-log assoc :python events)
(save-log-to-transform-run-message! run-id message-log)
(recur))))
(= 404 status)
@@ -110,213 +69,24 @@
(log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))
(defn- open-python-message-update-future! ^Closeable [run-id message-log]
- (let [cleanup (fn [fut]
- (future-cancel fut)
- (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
- (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
- (log/debugf "Log polling task done, run-id: %s" run-id)))
- fut (u.jvm/in-virtual-thread*
- (python-message-update-loop! run-id message-log))]
- (reify Closeable
- (close [_] (cleanup fut)))))
-
-(defn- table-schema [table-name metadata]
- {:name (if (keyword? table-name) table-name (keyword table-name))
- :columns (mapv (fn [{:keys [name base_type #_database_type]}]
- {:name name
- :type (python-runner/restricted-insert-type base_type)
- ;; :database-type database_type
- :nullable? true})
- (:fields metadata))})
-
-(defn- insert-data!
- "Insert data from source into an existing table."
- [driver db-id table-schema data-source]
- (let [data-source (assoc data-source :table-schema table-schema)]
- (driver/insert-from-source! driver db-id table-schema data-source)))
-
-(defn- create-table-and-insert-data!
- "Create a table from metadata and insert data from source."
- [driver db-id table-schema data-source]
- (transforms.util/create-table-from-schema! driver db-id table-schema)
- (insert-data! driver db-id table-schema data-source))
-
-(defn- transfer-with-rename-tables-strategy!
- "Transfer data using the rename-tables*! multimethod with atomicity guarantees.
- Creates new table, then atomically renames target->old and new->target, then drops old."
- [driver db-id table-name metadata data-source]
- (let [source-table-name (driver.u/temp-table-name driver table-name)
- temp-table-name (driver.u/temp-table-name driver table-name)]
- (log/info "Using rename-tables strategy with atomicity guarantees")
- (try
-
- (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
- (transforms.util/rename-tables! driver db-id {table-name temp-table-name
- source-table-name table-name})
- (transforms.util/drop-table! driver db-id temp-table-name)
-
- (catch Exception e
- (log/error e "Failed to transfer data using rename-tables strategy")
- (try
- (transforms.util/drop-table! driver db-id source-table-name)
- (catch Exception _))
- (throw e)))))
-
-(defn- transfer-with-create-drop-rename-strategy!
- "Transfer data using create + drop + rename to minimize time without data.
- Creates new table, drops old table, then renames new->target."
- [driver db-id table-name metadata data-source]
- (let [source-table-name (driver.u/temp-table-name driver table-name)]
- (log/info "Using create-drop-rename strategy to minimize downtime")
- (try
-
- (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)
- (transforms.util/drop-table! driver db-id table-name)
- (driver/rename-table! driver db-id source-table-name table-name)
-
- (catch Exception e
- (log/error e "Failed to transfer data using create-drop-rename strategy")
- (try
- (transforms.util/drop-table! driver db-id source-table-name)
- (catch Exception _))
- (throw e)))))
-
-(defn- transfer-with-drop-create-fallback-strategy!
- "Transfer data using drop + create fallback strategy.
- Drops old table, then creates new table directly with target name."
- [driver db-id table-name metadata data-source]
- (log/info "Using drop-create fallback strategy")
- (try
-
- (transforms.util/drop-table! driver db-id table-name)
- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)
-
- (catch Exception e
- (log/error e "Failed to transfer data using drop-create fallback strategy")
- (throw e))))
-
-(defmulti ^:private transfer-file-to-db
- {:arglists '([driver db transform metadata temp-file])}
- (fn [_ _ transform _ _] (-> transform :target :type keyword)))
-
-(defmethod transfer-file-to-db :table-incremental
- [driver {db-id :id}
- {:keys [target] :as transform}
- metadata temp-file]
- (let [table-name (transforms.util/qualified-table-name driver target)
- table-exists? (transforms.util/target-table-exists? transform)
- data-source {:type :jsonl-file
- :file temp-file}]
-
- ;; once we have more than just append, dispatch on :target-incremental-strategy
-
- (if (not table-exists?)
- (do
- (log/info "New table")
- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
- (insert-data! driver db-id (table-schema table-name metadata) data-source))))
-
-(defmethod transfer-file-to-db :table
- [driver {db-id :id :as db}
- {:keys [target] :as transform}
- metadata temp-file]
- (let [table-name (transforms.util/qualified-table-name driver target)
- table-exists? (transforms.util/target-table-exists? transform)
- data-source {:type :jsonl-file
- :file temp-file}]
- (cond
- (not table-exists?)
- (do
- (log/info "New table")
- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))
-
- (driver.u/supports? driver :atomic-renames db)
- (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)
-
- (driver.u/supports? driver :rename db)
- (transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)
-
- :else
- (transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))
-
-(defn- start-cancellation-process!
- "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.
- Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."
- [server-url run-id cancel-chan]
- (a/go (when (a/<! cancel-chan)
- (python-runner/cancel-python-code-http-call! server-url run-id))))
-
-(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
- ;; Resolve name-based source table refs to table IDs (throws if any not found)
- (let [resolved-source-tables (transforms.util/resolve-source-tables (:source-tables source))]
- ;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
- (with-open [^Closeable log-future-ref
- (if (app-db/in-transaction?)
- ;; if in a transaction (such as under mt/with-temp), it is not safe to poll for logs (close race / contention)
- ;; tests that want to test async log behaviour should opt out of thread-local test helpers
- (reify Closeable (close [_]))
- (open-python-message-update-future! run-id message-log))
- shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
- (let [driver (:engine db)
- server-url (transforms-python.settings/python-runner-url)
- _ (python-runner/copy-tables-to-s3! {:run-id run-id
- :shared-storage @shared-storage-ref
- :source (assoc source :source-tables resolved-source-tables)
- :cancel-chan cancel-chan
- :limit (:limit source)
- :transform-id (:id transform)})
- _ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan
- {:keys [status body] :as response}
- (python-runner/execute-python-code-http-call!
- {:server-url server-url
- :code (:body source)
- :run-id run-id
- :table-name->id resolved-source-tables
- :shared-storage @shared-storage-ref})
-
- output-manifest (python-runner/read-output-manifest @shared-storage-ref)
- events (python-runner/read-events @shared-storage-ref)]
- (.close ^Closeable log-future-ref) ; early close to force any writes to flush
- (replace-python-logs! message-log events)
- (if (not= 200 status)
- (do
- (when (:timeout body)
- (transforms/timeout-run! run-id))
- (throw (ex-info "Python runner call failed"
- {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
- :status-code 400
- :api-status-code status
- :body body
- :events events})))
- (try
- (let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
- temp-file (.toFile temp-path)]
- (when-not (seq (:fields output-manifest))
- (throw (ex-info "No fields in metadata"
- {:metadata output-manifest
- :raw-body body
- :events events})))
- (try
- (with-open [in (python-runner/open-output @shared-storage-ref)]
- (io/copy in temp-file))
- (let [file-size (.length temp-file)]
- (transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]]
- (transfer-file-to-db driver db transform output-manifest temp-file))
- (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
- (finally
- (.delete temp-file))))
- response
- (catch Exception e
- (log/error e "Failed to to create resulting table")
- (throw (ex-info "Failed to create the resulting table"
- {:transform-message (or (:transform-message (ex-data e))
- ;; TODO keeping messaging the same at this level
- ;; should be more specific in underlying calls
- (i18n/tru "Failed to create the resulting table"))}
- e)))))))))
+ (if (app-db/in-transaction?)
+ ;; in a transaction (such as under mt/with-temp), it is not safe to poll for logs
+ (do
+ (log/warnf "Log polling disabled as we are within a database transaction, run-id: %s" run-id)
+ (reify Closeable (close [_])))
+ ;; poll updates until execution is complete
+ (let [cleanup (fn [fut]
+ (future-cancel fut)
+ (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
+ (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
+ (log/debugf "Log polling task done, run-id: %s" run-id)))
+ fut (u.jvm/in-virtual-thread*
+ (python-message-update-loop! run-id message-log))]
+ (reify Closeable
+ (close [_] (cleanup fut))))))
(defn- exceptional-run-message [message-log ex]
- (str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)
+ (str/join "\n" (remove str/blank? [(base/message-log->string message-log)
(or (:transform-message (ex-data ex))
(if (instance? InterruptedException ex)
(i18n/tru "Transform interrupted")
@@ -327,43 +97,43 @@
Blocks until the transform returns."
[transform {:keys [run-method start-promise user-id]}]
- (assert (transforms.util/python-transform? transform) "Transform must be a python transform")
(try
- (let [message-log (empty-message-log)
+ (let [message-log (base/empty-message-log)
{:keys [target owner_user_id creator_id] transform-id :id} transform
- {driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))
- ;; For manual runs, use the triggering user; for cron, use owner/creator
+ {driver :engine :as db} (t2/select-one :model/Database (transforms-base.i/target-db-id transform))
run-user-id (if (and (= run-method :manual) user-id)
user-id
(or owner_user_id creator_id))
- {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]
+ {run-id :id} (transforms.u/try-start-unless-already-running transform-id run-method run-user-id)]
(some-> start-promise (deliver [:started run-id]))
- (log! message-log (i18n/tru "Executing Python transform"))
- (driver.conn/with-write-connection
- (log/info "Executing Python transform" transform-id "with target" (pr-str target)
- (when (driver.conn/write-connection-requested?)
- " using write connection"))
- (let [start-ms (u/start-timer)
- conn-spec (driver/connection-spec driver db)
- transform-details {:db-id (:id db)
- :transform-id transform-id
- :transform-type (keyword (:type target))
- :conn-spec conn-spec
- :output-schema (:schema target)
- :output-table (transforms.util/qualified-table-name driver target)}
- run-fn (fn [cancel-chan]
- (run-python-transform! transform db run-id cancel-chan message-log)
- (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))
- (save-log-to-transform-run-message! run-id message-log))
- ex-message-fn #(exceptional-run-message message-log %)
- result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
- (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
- (transforms.util/handle-transform-complete!
- :run-id run-id
- :transform transform
- :db db)
- {:run_id run-id
- :result result})))
+ (with-open [_ (open-python-message-update-future! run-id message-log)]
+ (driver.conn/with-write-connection
+ (let [conn-spec (driver/connection-spec driver db)
+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
+ run-fn (fn [cancel-chan]
+ (let [result (transforms-base.i/execute-base!
+ transform
+ {:cancelled? #(boolean (a/poll! cancel-chan))
+ :cancel-chan cancel-chan
+ :run-id run-id
+ :message-log message-log
+ :with-stage-timing-fn (fn [rid stage thunk]
+ (transforms.instrumentation/with-stage-timing [rid stage]
+ (thunk)))})]
+ (save-log-to-transform-run-message! run-id message-log)
+ (when-not (= :succeeded (:status result))
+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
+ result))
+ ex-message-fn #(exceptional-run-message message-log %)
+ result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
+ (transforms.u/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]
+ ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)
+ (transforms-base.u/complete-execution! transform
+ {:run-id run-id
+ :with-stage-timing-fn (fn [rid stage thunk]
+ (transforms.instrumentation/with-stage-timing [rid stage]
+ (thunk)))})
+ {:run_id run-id :result result}))))
(catch Throwable t
(log/error t "Error executing Python transform")
(throw t))))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
index 64cf330c203..bb23a6940ab 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj
@@ -3,37 +3,7 @@
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase.transforms.interface :as transforms.i]))
-(defmethod transforms.i/target-db-id :python
- [transform]
- (-> transform :target :database))
-
-(defmethod transforms.i/source-db-id :python
- [transform]
- (-> transform :source :source-database))
-
#_{:clj-kondo/ignore [:discouraged-var]}
(defmethod transforms.i/execute! :python
[transform options]
(transforms-python.execute/execute-python-transform! transform options))
-
-(defn- source-table-value->dependency
- "Convert a source table value (int or ref map) to a dependency map."
- [v]
- (cond
- ;; Integer table ID - direct table dependency
- (int? v)
- {:table v}
-
- ;; Map with resolved table_id - direct table dependency
- (:table_id v)
- {:table (:table_id v)}
-
- ;; Map without table_id - return ref for ordering system to resolve
- :else
- {:table-ref (select-keys v [:database_id :schema :table])}))
-
-(defmethod transforms.i/table-dependencies :python
- [transform]
- (into #{}
- (map source-table-value->dependency)
- (vals (get-in transform [:source :source-tables]))))
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
index 9432ddc6960..49691b82aa6 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj
@@ -14,8 +14,8 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
[metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.instrumentation :as transforms.instrumentation]
- [metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
[metabase.util.json :as json]
@@ -277,7 +277,7 @@
transform (t2/select-one :model/Transform transform-id)]
(cond-> (-> (lib/query metadata-provider table-metadata)
lib/disable-default-limit)
- source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform))
+ source-incremental-strategy (transforms-base.u/preprocess-incremental-query source-incremental-strategy (transforms-base.u/next-checkpoint transform))
limit (lib/limit limit))))
;; TODO break this up such that s3 can be swapped out for other transfer mechanisms.
@@ -294,6 +294,7 @@
(> (count (:source-tables source)) 1))
(throw (ex-info "Incremental transforms for python only supports one source table" {})))
(doseq [[table-name v] (:source-tables source)
+ ;; TODO (Chris 2026-03-03) We should also handle the case where table-id has not been backfilled yet...
:let [table-id (if (int? v) v (:table_id v))
{:keys [s3-client bucket-name objects]} shared-storage
{data-path :path} (get objects [:table table-id :data])
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
index 557dd408005..6df753785a3 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj
@@ -29,7 +29,7 @@
[metabase.request.core :as request]
[metabase.transforms.core :as transforms]
[metabase.transforms.feature-gating :as transforms.gating]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.i18n :as i18n :refer [deferred-tru tru]]
[metabase.util.malli.registry :as mr]
@@ -79,7 +79,7 @@
[:python
[:map {:closed true}
[:source-database {:optional true} :int]
- [:source-tables [:map-of [:string {:min 1}] :int]]
+ [:source-tables [:map-of [:string {:min 1}] [:or :int :map]]]
[:type [:= "python"]]
[:body :string]]]])
@@ -801,7 +801,7 @@
(doseq [field [:name :source :target]]
(api/check-400 (get body field) (str (name field) " is required when creating a new transform")))
;; Check premium feature requirements
- (api/check (transforms.util/check-feature-enabled body)
+ (api/check (transforms.u/check-feature-enabled body)
[402 (deferred-tru "Premium features required for this transform type are not enabled.")])
(t2/with-transaction [_tx]
(let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id ws-id))
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/common.clj
index 9a7d4eddeb2..6df7b59b6f0 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/common.clj
@@ -6,7 +6,7 @@
[metabase-enterprise.workspaces.util :as ws.u]
[metabase.api-keys.core :as api-key]
[metabase.api.common :as api]
- [metabase.transforms.interface :as transforms.i]
+ [metabase.transforms-base.interface :as transforms-base.i]
[metabase.util.log :as log]
[metabase.util.quick-task :as quick-task]
[toucan2.core :as t2]))
@@ -157,7 +157,7 @@
(ws.u/assert-transform! entity-type)
;; Initialize workspace if uninitialized (outside transaction so async task can see committed data)
(let [workspace (if (= :uninitialized (:db_status workspace))
- (let [target-db-id (transforms.i/target-db-id body)]
+ (let [target-db-id (transforms-base.i/target-db-id body)]
(api/check-400 target-db-id "Transform must have a target database")
(initialize-workspace! workspace target-db-id))
workspace)]
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
index 5066efea514..c97c323abc1 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj
@@ -175,13 +175,25 @@
(inputs-from-native-query query db-id)
(inputs-from-mbql-query query)))
+(defn- source-table-ref->table-ref
+ "Paper over a small unfortunate difference in shape"
+ [table-ref]
+ (-> (select-keys table-ref [:schema :table :table_id])
+ (assoc :db_id (:database_id table-ref))))
+
(defn- inputs-from-python-transform
"Extract table refs from a python transform's source-tables.
- Python transforms require tables to exist (they map name -> table_id). Batch lookup."
+ Uses metadata from map refs directly; only looks up the database for bare integer IDs."
[source-tables]
- (let [table-ids (set (vals source-tables))
- table-refs (batch-table-refs-from-ids table-ids)]
- (u/keepv table-refs (vals source-tables))))
+ (let [{maps true ints false} (group-by (comp map? val) source-tables)
+ ;; Maps already have schema/table metadata
+ map-refs (map (comp source-table-ref->table-ref val) maps)
+ ;; Integers need a batch lookup
+ int-ids (map val ints)
+ int-refs (when (seq int-ids)
+ (let [lookup (batch-table-refs-from-ids int-ids)]
+ (keep lookup int-ids)))]
+ (into (vec map-refs) int-refs)))
(mu/defn analyze-entity :- ::analysis
"Analyze a workspace entity to find its dependencies.
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
index cce652ab593..ac636ef9e8f 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj
@@ -1,37 +1,20 @@
(ns metabase-enterprise.workspaces.execute
- "This namespace is concerned with executing non-AppDB transforms, with optional reference re-mappings.
+ "Executing transforms with optional reference re-mappings, without writing to AppDB.
- It currently lives inside the workspace module, but eventually will become part of the transform module, so it should
- not make any assumptions about being run within a workspace, having the input / output semantics of a workspace, and
- especially it should not touch any of the Workspace entities inside AppDb, or any of the other workspace namespaces.
-
- For now, it uses AppDB as a side-channel with the transforms module, but in future this module should be COMPLETELY
- decoupled from AppDb."
+ Uses transforms-base.core/execute! to run transforms in-memory. No Transform or TransformRun
+ rows are created in AppDB. Warehouse DB changes (actual table data) persist in the isolated schema."
(:require
[clojure.string :as str]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
- [metabase.api.common :as api]
[metabase.query-processor :as qp]
[metabase.sql-tools.core :as sql-tools]
- [metabase.transforms.core :as transforms]
- [metabase.transforms.execute :as transforms.execute]
- [metabase.transforms.util :as transforms.util]
- [metabase.util :as u]
+ [metabase.transforms-base.core :as transforms-base]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util.log :as log]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
-(defn- execution-results
- "Extract execution metadata from transform_run and target table info.
- Must be called within the transaction before rollback."
- [{xf-id :id :keys [target]}]
- (let [run (t2/select-one :model/TransformRun :transform_id xf-id)]
- (merge
- (select-keys run [:status :start_time :end_time :message])
- {:table {:name (:name target)
- :schema (:schema target)}})))
-
(defn- remap-python-source
"Remap source-tables in a Python transform source to point to isolated tables.
Handles both legacy integer format and new map format (from PR #66934)."
@@ -113,7 +96,7 @@
[{:keys [source]} remapping]
(let [table-mapping (:tables remapping no-mapping)
remapped-source (remap-python-source table-mapping source)
- resolved-source-tables (transforms.util/resolve-source-tables (:source-tables remapped-source))
+ resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables remapped-source))
{:as result
:keys [cols rows]} (python-runner/execute-and-read-output!
{:code (:body remapped-source)
@@ -161,7 +144,7 @@
"Run SQL transform query and return first 2000 rows without persisting.
Returns a ::ws.t/query-result map with data nested under :data to match /api/dataset format."
[{:keys [source]} remapping]
- (run-query source (transforms/transform-source-type source) remapping
+ (run-query source (transforms-base.u/transform-source-type source) remapping
{:error-context "Failed to run sql dry-run"}))
(defn execute-adhoc-sql
@@ -182,46 +165,35 @@
"Execute transform and return first 2000 rows without persisting.
Returns a ::ws.t/query-result map with data nested under :data."
[{:keys [source] :as transform} remapping]
- (let [s-type (transforms/transform-source-type source)]
+ (let [s-type (transforms-base.u/transform-source-type source)]
(case s-type
(:native :mbql) (dry-run-sql transform remapping)
:python (dry-run-python transform remapping))))
(defn run-transform-with-remapping
- "Execute a given collection with the given table and field re-mappings.
+ "Execute a given transform with the given table and field re-mappings.
This is used by Workspaces to re-route the output of each transform to a non-production table, and
to re-write their queries where these outputs transitively become inputs to other transforms.
Returns an ::ws.t/execution-result map with status, timing, and table metadata.
- -------
-
- NOTE: currently execution is done using transaction-rollback pattern, as a short-term hack.
-
- 1. Creating temporary Transform/TransformRun records in a transaction
- 2. Executing using existing transform infrastructure
- 3. Scraping metadata (execution stats + table schema)
- 4. Rolling back the transaction (no app DB records persist)
-
- The warehouse DB changes (actual table data) DO persist in the isolated schema."
+ Uses transforms-base.core/execute! to run the transform in-memory without writing
+ Transform or TransformRun rows to AppDB. The warehouse DB changes (actual table data)
+ DO persist in the isolated schema."
[{:keys [source target] :as transform} remapping]
- (t2/with-transaction [_conn]
- (let [s-type (transforms/transform-source-type source)
- table-mapping (:tables remapping no-mapping)
- target-fallback (:target-fallback remapping no-mapping)
- field-mapping (:fields remapping no-mapping)
- new-xf (-> (select-keys transform [:name :description])
- (assoc :creator_id api/*current-user-id*
- :source (remap-source table-mapping field-mapping s-type source)
- :target (remap-target table-mapping target-fallback target)))
- _ (assert (:target new-xf) "Target mapping must not be nil")
- temp-xf (t2/insert-returning-instance! :model/Transform new-xf)]
- (try
- (transforms.execute/execute! temp-xf {:run-method :manual})
- (catch Exception _e
- ;; Execution failed - the TransformRun record has been updated with failure status and message
- nil))
- ;; Return execution results whether succeeded or failed
- (u/prog1 (execution-results temp-xf)
- (t2/delete! :model/Transform (:id temp-xf))))))
+ (let [s-type (transforms-base.u/transform-source-type source)
+ table-mapping (:tables remapping no-mapping)
+ target-fallback (:target-fallback remapping no-mapping)
+ field-mapping (:fields remapping no-mapping)
+ remapped-xf (-> (select-keys transform [:name :description])
+ (assoc :source (remap-source table-mapping field-mapping s-type source)
+ :target (remap-target table-mapping target-fallback target)))
+ _ (assert (:target remapped-xf) "Target mapping must not be nil")
+ start-time (java.time.Instant/now)
+ result (transforms-base/execute! remapped-xf {:publish-events? false})]
+ {:status (:status result)
+ :start_time start-time
+ :end_time (java.time.Instant/now)
+ :message (or (:logs result) (some-> (:error result) ex-message))
+ :table (select-keys (:target remapped-xf) [:name :schema])}))
diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.clj b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.clj
index 59a72030f4b..58b66c28224 100644
--- a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.clj
+++ b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.clj
@@ -3,10 +3,8 @@
and edited within a workspace."
(:require
[clojure.string :as str]
- [medley.core :as m]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
[metabase.models.interface :as mi]
+ [metabase.models.transforms.transform :as transform]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -24,25 +22,9 @@
(derive :hook/timestamped?)
(derive :hook/entity-id))
-;; TODO (Chris 2025-12-11) -- we need to share a bunch of stuff with transforms, i think we'll need to reorganize modules
-;; suggestion: add a transforms-interfaces module which both transforms and workspaces depend on.
-
-(defn- transform-source-out-DUPLICATED [m]
- (-> m
- mi/json-out-without-keywordization
- (update-keys keyword)
- (m/update-existing :query lib-be/normalize-query)
- (m/update-existing :type keyword)
- (m/update-existing :source-incremental-strategy #(update-keys % keyword))))
-
-(defn- transform-source-in-DUPLICATED [m]
- (-> m
- (m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))
- mi/json-in))
-
(t2/deftransforms :model/WorkspaceTransform
{:ref_id {:in identity :out str/trim}
- :source {:out transform-source-out-DUPLICATED, :in transform-source-in-DUPLICATED}
+ :source {:out transform/transform-source-out, :in transform/transform-source-in}
:target mi/transform-json})
(t2/define-before-insert :model/WorkspaceTransform
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index 550aa9bae4d..a02eb8d6dcb 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -15,10 +15,10 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor :as qp]
[metabase.test :as mt]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.test-dataset :as transforms-dataset]
[metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
- [metabase.transforms.util :as transforms.u]
[next.jdbc :as next.jdbc]
[next.jdbc.result-set :as jdbc.rs]
[toucan2.core :as t2]))
@@ -183,8 +183,8 @@
(some-> result :data :rows first first bigint)))
(defn get-checkpoint-value [transform]
- (#'transforms.u/next-checkpoint-value
- (transforms.u/next-checkpoint transform)))
+ (#'transforms-base.u/next-checkpoint-value
+ (transforms-base.u/next-checkpoint transform)))
(defn- compare-checkpoint-values
"Compare two checkpoint values with type-appropriate logic. "
@@ -609,7 +609,7 @@
(testing "sync has picked up table"
(is (=? {:name target-table, :fields [{:name "id"}]} (-> (t2/select-one :model/Table :name target-table) (t2/hydrate :fields)))))
(testing "checkpoint is recognized"
- (is (some? (transforms.u/next-checkpoint transform))))))))))
+ (is (some? (transforms-base.u/next-checkpoint transform))))))))))
(deftest checkpoint-field-does-not-exist-test
(mt/test-drivers #{:postgres} ; no db specifics
@@ -636,7 +636,7 @@
(testing "target table has expected data"
(is (= [{:id 42}] (pg-table-rows db-spec target-table))))
(testing "checkpoint is not recognized, so transform acts as if no checkpoint"
- (is (nil? (transforms.u/next-checkpoint transform))))
+ (is (nil? (transforms-base.u/next-checkpoint transform))))
;; Maybe this is unrealistic - you cannot select a column that does not exist
;; But the source tables schema can change, e.g. rename: you change event_time to event_ts or something
;; so one would have to be careful to disable or delete transforms ahead of a schema change like this
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
index 2fc119cbb36..f6a81f187a7 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj
@@ -2,8 +2,8 @@
(:require
[clojure.test :refer [deftest testing is]]
[metabase.test :as mt]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.ordering :as ordering]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.ordering :as ordering]
[toucan2.core :as t2]))
(defn- make-python-transform
@@ -34,7 +34,7 @@
:table "intermediate_output"}}
"final_output")]
(testing "table-dependencies returns table-ref for unresolved name reference"
- (let [deps (transforms.i/table-dependencies (t2/select-one :model/Transform :id t-b))]
+ (let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))]
(is (contains? deps {:table-ref {:database_id (mt/id)
:schema "public"
:table "intermediate_output"}}))))
@@ -58,7 +58,7 @@
:table "output_a"}}
"output_b")]
(testing "table-dependencies includes both types"
- (let [deps (transforms.i/table-dependencies (t2/select-one :model/Transform :id t-b))]
+ (let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))]
(is (contains? deps {:table (mt/id :products)}))
(is (contains? deps {:table-ref {:database_id (mt/id)
:schema "public"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
index dde770706d3..27d8ece4dca 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj
@@ -3,7 +3,7 @@
[clojure.test :refer [deftest testing is]]
[metabase.driver.util :as driver.u]
[metabase.test :as mt]
- [metabase.transforms.util :as transforms.util]))
+ [metabase.transforms.util :as transforms.u]))
(set! *warn-on-reflection* true)
@@ -12,16 +12,16 @@
(let [table-with-schema {:name (name (driver.u/temp-table-name :postgres :schema/orders))}
table-without-schema {:name (name (driver.u/temp-table-name :postgres :orders))}]
(mt/with-premium-features #{:hosting}
- (is (false? (transforms.util/is-temp-transform-table? table-with-schema)))
- (is (false? (transforms.util/is-temp-transform-table? table-without-schema))))
+ (is (false? (transforms.u/is-temp-transform-table? table-with-schema)))
+ (is (false? (transforms.u/is-temp-transform-table? table-without-schema))))
(mt/with-premium-features #{}
- (is (transforms.util/is-temp-transform-table? table-without-schema))
- (is (transforms.util/is-temp-transform-table? table-with-schema)))
+ (is (transforms.u/is-temp-transform-table? table-without-schema))
+ (is (transforms.u/is-temp-transform-table? table-with-schema)))
(mt/with-premium-features #{:hosting :transforms}
- (is (transforms.util/is-temp-transform-table? table-without-schema))
- (is (transforms.util/is-temp-transform-table? table-with-schema)))))
+ (is (transforms.u/is-temp-transform-table? table-without-schema))
+ (is (transforms.u/is-temp-transform-table? table-with-schema)))))
(testing "Ignores non-transform tables"
(mt/with-premium-features #{:transforms}
- (is (false? (transforms.util/is-temp-transform-table? {:name :orders})))
- (is (false? (transforms.util/is-temp-transform-table? {:name :public/orders}))))))
+ (is (false? (transforms.u/is-temp-transform-table? {:name :orders})))
+ (is (false? (transforms.u/is-temp-transform-table? {:name :public/orders}))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
index ba7bce4dd3c..606a8789afe 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj
@@ -12,8 +12,8 @@
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
- [metabase.transforms.util :as transforms.util]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.json :as json]
@@ -121,7 +121,7 @@
table-schema {:name qualified-table-name
:columns columns}]
(mt/as-admin
- (transforms.util/create-table-from-schema! driver db-id table-schema))
+ (transforms-base.u/create-table-from-schema! driver db-id table-schema))
(when (seq data)
(driver/insert-from-source! driver db-id table-schema
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
index fd97387be84..4fe6138a118 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
@@ -4,9 +4,10 @@
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase.test :as mt]
[metabase.test.util :as test.util]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.test-dataset :as transforms-dataset]
[metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms.util :as transforms.u]
[toucan2.core :as t2])
(:import
(java.util.concurrent CountDownLatch)))
@@ -48,10 +49,10 @@
" return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))
(let [swap-latch (CountDownLatch. 1)
- original-rename-tables-atomic! transforms.util/rename-tables!]
- (with-redefs [transforms.util/rename-tables! (fn [driver db-id rename-pairs]
- (.await swap-latch)
- (original-rename-tables-atomic! driver db-id rename-pairs))]
+ original-rename-tables-atomic! transforms-base.u/rename-tables!]
+ (mt/with-dynamic-fn-redefs [transforms-base.u/rename-tables! (fn [driver db-id rename-pairs]
+ (.await swap-latch)
+ (original-rename-tables-atomic! driver db-id rename-pairs))]
(let [transform-future (future
(transforms-python.execute/execute-python-transform!
(t2/select-one :model/Transform (:id transform))
@@ -91,7 +92,7 @@
(let [db-id (mt/id)
tables (t2/select :model/Table :db_id db-id :active true)]
- (is (not-any? transforms.util/is-temp-transform-table? tables)
+ (is (not-any? transforms.u/is-temp-transform-table? tables)
"No temp tables should remain after successful Python transform")
(is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
index 177bc41dee5..af3c34383e4 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj
@@ -59,9 +59,9 @@
:name "Test Python Transform"}
run-id 101
logged-messages (atom [])]
- (with-redefs [log/log* (fn [_ level _ message]
- (swap! logged-messages conj {:level level :message message}))
- transform-run/running-run-for-transform-id (constantly nil)]
+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]
+ (swap! logged-messages conj {:level level :message message}))
+ transform-run/running-run-for-transform-id (constantly nil)]
(#'jobs/run-transform! run-id :scheduled nil python-transform)
(is (= 1 (count @logged-messages))
"Should log exactly one warning")
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
index 785d3639815..441ef4ab21a 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj
@@ -2,8 +2,8 @@
(:require
[clojure.test :refer :all]
[metabase.test :as mt]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.ordering :as ordering]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.ordering :as ordering]
[metabase.transforms.test-util :as transforms.tu]
[toucan2.core :as t2]))
@@ -33,7 +33,7 @@
(defn- transform-deps-for-db [transform]
(mt/with-metadata-provider (mt/id)
- (#'transforms.i/table-dependencies transform)))
+ (transforms-base.i/table-dependencies transform)))
(deftest python-transform-basic-dependencies-test
(testing "Python transforms with source-tables dependencies are extracted correctly"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
index 5b1fc2ae7d5..e6532cb20a8 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj
@@ -15,7 +15,7 @@
[metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
[metabase.test.util :as tu]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util :as u]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -395,11 +395,11 @@
(deftest transform-function-without-libraries-test
(testing "transform function works when no libraries exist"
(mt/test-drivers #{:postgres}
- (with-redefs [t2/select-fn->fn (fn [k v model]
- (when (and (= k :path)
- (= v :source)
- (= model :model/PythonLibrary))
- {}))]
+ (mt/with-dynamic-fn-redefs [t2/select-fn->fn (fn [k v model]
+ (when (and (= k :path)
+ (= v :source)
+ (= model :model/PythonLibrary))
+ {}))]
(let [transform-code (str "import pandas as pd\n"
"\n"
"def transform():\n"
@@ -412,11 +412,11 @@
(deftest transform-function-library-import-error-test
(testing "transform function handles missing library gracefully"
(mt/test-drivers #{:postgres}
- (with-redefs [t2/select-fn->fn (fn [k v model]
- (when (and (= k :path)
- (= v :source)
- (= model :model/PythonLibrary))
- {"utils" "def helper():\n return 42"}))]
+ (mt/with-dynamic-fn-redefs [t2/select-fn->fn (fn [k v model]
+ (when (and (= k :path)
+ (= v :source)
+ (= model :model/PythonLibrary))
+ {"utils" "def helper():\n return 42"}))]
(let [transform-code (str "import pandas as pd\n"
"from common import some_function # This library doesn't exist\n"
"\n"
@@ -452,7 +452,7 @@
{:name "created_date" :type :type/Date :nullable? true}
{:name "description" :type :type/Text :nullable? true}]}
_ (mt/as-admin
- (transforms.util/create-table-from-schema! driver db-id table-schema))
+ (transforms-base.u/create-table-from-schema! driver db-id table-schema))
row-values [[1
19.99
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
index 23fbf41e020..a5cba2543be 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase-enterprise.transforms-python.base :as transforms-python.base]
[metabase-enterprise.transforms-python.execute :as transforms-python.execute]
[metabase-enterprise.transforms-python.init]
[metabase-enterprise.transforms-python.python-runner :as transforms-python.python-runner]
@@ -237,9 +238,9 @@
(run-scenario [scenario schema]
(with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)
- transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
+ transforms-python.base/transfer-file-to-db (if-some [e (:writeback-ex scenario)]
(fn [& _] (throw e))
- @#'transforms-python.execute/transfer-file-to-db)]
+ @#'transforms-python.base/transfer-file-to-db)]
(with-transform-cleanup! [target {:type "table"
:schema schema
:name "result"}]
@@ -383,7 +384,7 @@
(fn [os fields-meta col-meta]
(rf-proxy ready-signal wait-signal (f os fields-meta col-meta)))})
:write
- (let [f-ref #'transforms-python.execute/transfer-file-to-db
+ (let [f-ref #'transforms-python.base/transfer-file-to-db
f @f-ref]
{f-ref
(fn [& args]
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
index a056b3214ae..943f05ef1b4 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj
@@ -316,12 +316,12 @@
(testing "No updates are propagated back to core app on merge failure"
(let [update-transform! transforms.core/update-transform!]
- (with-redefs [transforms.core/update-transform! (let [call-count (atom 0)]
- (fn [& args]
- (when (> @call-count 0)
- (throw (Exception. "boom")))
- (swap! call-count inc)
- (apply update-transform! args)))]
+ (mt/with-dynamic-fn-redefs [transforms.core/update-transform! (let [call-count (atom 0)]
+ (fn [& args]
+ (when (> @call-count 0)
+ (throw (Exception. "boom")))
+ (swap! call-count inc)
+ (apply update-transform! args)))]
(testing "API response: empty merged, single error"
(let [resp (mt/user-http-request :crowberto :post 200 (ws-url ws-id "/merge"))]
(is (empty? (get-in resp [:merged :transforms])))
@@ -574,8 +574,8 @@
_ (mt/user-http-request :crowberto :post 204
(ws-url ws-id "/transform" ws-x-1-id "/archive"))]
(testing "Failure on merge"
- (with-redefs [transforms.core/delete-transform! (fn [& _args]
- (throw (Exception. "boom")))]
+ (mt/with-dynamic-fn-redefs [transforms.core/delete-transform! (fn [& _args]
+ (throw (Exception. "boom")))]
(let [resp (mt/user-http-request :crowberto :post 500
(ws-url ws-id "/transform" ws-x-1-id "/merge"))]
(testing "Response"
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
index 591602563e8..7d105fdcf60 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj
@@ -55,9 +55,8 @@
:isolated_table_id number?}]
(t2/select :model/WorkspaceOutput :workspace_id (:id workspace) :ref_id (:ref_id ws-transform))))))
- (testing "app DB records are rolled back"
- ;; TransformRun is +1 because cascade delete was removed in d1e940e66b5
- (is (= (update before :xfrun inc)
+ (testing "no app DB records are created (transforms-base/execute! skips TransformRun)"
+ (is (= before
{:xf (t2/count :model/Transform)
:xfrun (t2/count :model/TransformRun)})))))))
diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts
index 020affdbe2a..ae8b6e10a2e 100644
--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts
+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts
@@ -3,6 +3,7 @@ import { useMemo } from "react";
import { useGetTransformQuery } from "metabase/api";
import { useSelector } from "metabase/lib/redux";
import { getMetadata } from "metabase/selectors/metadata";
+import { collapseSourceTableReferences } from "metabase/transforms/utils";
import { useGetWorkspaceTransformQuery } from "metabase-enterprise/api";
import type {
PythonTransformSource,
@@ -78,12 +79,14 @@ export function useTransformSources(
return {
oldSource,
oldSourceTables: isPythonTransformSource(globalTransform?.source)
- ? globalTransform.source["source-tables"]
+ ? collapseSourceTableReferences(globalTransform.source["source-tables"])
: undefined,
oldTarget: globalTransform?.target,
newSource,
newSourceTables: isPythonTransformSource(workspaceTransform?.source)
- ? workspaceTransform.source["source-tables"]
+ ? collapseSourceTableReferences(
+ workspaceTransform.source["source-tables"],
+ )
: undefined,
newTarget: workspaceTransform?.target,
hasError,
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
index 5b252faeda2..71f035beee5 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx
@@ -1,10 +1,12 @@
-import { useEffect, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
import { t } from "ttag";
import { skipToken, useListTablesQuery } from "metabase/api";
import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";
+import { extractTableId } from "metabase/transforms/utils";
import { Box, Button, Icon, Stack, Text } from "metabase/ui";
import type {
+ ConcreteTableId,
DatabaseId,
PythonTransformTableAliases,
Table,
@@ -174,6 +176,13 @@ function SelectionInput({
}) {
const table = availableTables.find((table) => table.id === selection.tableId);
+ const selectedTableIds = useMemo((): ConcreteTableId[] => {
+ return Object.values(tables).flatMap((v) => {
+ const id = extractTableId(v);
+ return id != null ? [id] : [];
+ });
+ }, [tables]);
+
function handleAliasChange(newAlias: string) {
const newSelection = {
...selection,
@@ -218,7 +227,7 @@ function SelectionInput({
<TableSelector
database={database}
table={table}
- selectedTableIds={Object.values(tables)}
+ selectedTableIds={selectedTableIds}
onChange={handleTableChange}
onRemove={onRemove}
availableTables={availableTables}
diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
index 69666c2fc77..793ac59e6d2 100644
--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts
@@ -1,4 +1,5 @@
import { slugify as toSlug } from "metabase/lib/formatting";
+import { extractTableId } from "metabase/transforms/utils";
import type { PythonTransformTableAliases } from "metabase-types/api";
import type { TableSelection } from "./types";
@@ -7,8 +8,8 @@ export function getInitialTableSelections(
tables: PythonTransformTableAliases | undefined,
) {
if (tables && Object.keys(tables).length > 0) {
- return Object.entries(tables).map(([alias, tableId]) => ({
- tableId,
+ return Object.entries(tables).map(([alias, value]) => ({
+ tableId: extractTableId(value),
alias,
}));
}
diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.ts
index bd2e61ea97f..05a2e5592cf 100644
--- a/frontend/src/metabase/transforms/utils.ts
+++ b/frontend/src/metabase/transforms/utils.ts
@@ -8,9 +8,11 @@ import * as Lib from "metabase-lib";
import type Metadata from "metabase-lib/v1/metadata/Metadata";
import type {
CollectionNamespace,
+ ConcreteTableId,
Database,
DatabaseId,
DraftTransformSource,
+ PythonTransformTableAliases,
TemplateTag,
Transform,
TransformRun,
@@ -135,11 +137,78 @@ export function isSameSource(
return Lib.areLegacyQueriesEqual(source1.query, source2.query);
}
if (source1.type === "python" && source2.type === "python") {
- return _.isEqual(source1, source2);
+ return _.isEqual(
+ {
+ ...source1,
+ "source-tables": collapseSourceTableReferences(
+ source1["source-tables"],
+ ),
+ },
+ {
+ ...source2,
+ "source-tables": collapseSourceTableReferences(
+ source2["source-tables"],
+ ),
+ },
+ );
}
return false;
}
+/**
+ * Extract a ConcreteTableId from a source-tables value, which may be either
+ * a bare integer or a map with table_id (from backend normalization).
+ */
+export function extractTableId(
+ value: ConcreteTableId | Record<string, unknown>,
+): ConcreteTableId | undefined {
+ if (typeof value === "number") {
+ return value;
+ }
+ if (typeof value === "object" && value != null) {
+ const id = (value as Record<string, unknown>)["table_id"];
+ if (typeof id === "number") {
+ return id as ConcreteTableId;
+ }
+ }
+ return undefined;
+}
+
+const collapseCache = new WeakMap<
+ Record<string, unknown>,
+ PythonTransformTableAliases
+>();
+
+/**
+ * Collapse source-table references to plain { alias: tableId } form.
+ * The backend may return map refs (with table_id, schema, etc.) instead of
+ * bare integer IDs. Returns the original map unchanged if all values are
+ * already plain integers. Results are memoized by object reference.
+ */
+export function collapseSourceTableReferences(
+ tables: Record<string, unknown>,
+): PythonTransformTableAliases {
+ const cached = collapseCache.get(tables);
+ if (cached) {
+ return cached;
+ }
+ const values = Object.values(tables);
+ if (values.every((v) => typeof v === "number")) {
+ return tables as PythonTransformTableAliases;
+ }
+ const result: PythonTransformTableAliases = {};
+ for (const [key, value] of Object.entries(tables)) {
+ const id = extractTableId(
+ value as ConcreteTableId | Record<string, unknown>,
+ );
+ if (id != null) {
+ result[key] = id;
+ }
+ }
+ collapseCache.set(tables, result);
+ return result;
+}
+
export function isSourceEmpty(
source: DraftTransformSource,
databaseId: DatabaseId,
diff --git a/resources/python-sources/.lock b/resources/python-sources/.lock
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/src/metabase/collections_rest/api.clj b/src/metabase/collections_rest/api.clj
index 033ae6114bf..f0ee92bc1bd 100644
--- a/src/metabase/collections_rest/api.clj
+++ b/src/metabase/collections_rest/api.clj
@@ -28,7 +28,7 @@
[metabase.request.core :as request]
[metabase.revisions.core :as revisions]
[metabase.transforms.feature-gating :as transforms.gating]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms.util :as transforms.u]
[metabase.upload.core :as upload]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
@@ -488,7 +488,7 @@
(defmethod collection-children-query :transform
[_model collection {:keys [pinned-state]}]
- (let [enabled-types (transforms.util/enabled-source-types-for-user)]
+ (let [enabled-types (transforms.u/enabled-source-types-for-user)]
{:select [:id :collection_id :name [(h2x/literal "transform") :model] :description :entity_id]
:from [[:transform :transform]]
:where [:and
diff --git a/src/metabase/core/init.clj b/src/metabase/core/init.clj
index 3fc57750572..3e546ff3f53 100644
--- a/src/metabase/core/init.clj
+++ b/src/metabase/core/init.clj
@@ -65,6 +65,7 @@
[metabase.task-history.init]
[metabase.testing-api.init]
[metabase.tiles.init]
+ [metabase.transforms-base.init]
[metabase.transforms.init]
[metabase.types.init]
[metabase.upload.init]
diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.clj
index 531470506eb..53ec780df1d 100644
--- a/src/metabase/models/transforms/transform.clj
+++ b/src/metabase/models/transforms/transform.clj
@@ -15,8 +15,9 @@
[metabase.search.core :as search.core]
[metabase.search.ingestion :as search]
[metabase.search.spec :as search.spec]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.log :as log]
[methodical.core :as methodical]
@@ -33,8 +34,8 @@
(defmethod mi/can-read? :model/Transform
([instance]
(and (api/is-data-analyst?)
- (transforms.util/source-tables-readable? instance)
- (transforms.util/check-feature-enabled instance)))
+ (transforms.u/source-tables-readable? instance)
+ (transforms.u/check-feature-enabled instance)))
([_model pk]
(when-let [transform (t2/select-one :model/Transform :id pk)]
(mi/can-read? transform))))
@@ -60,10 +61,10 @@
;; Inline can-write? logic since instance is a plain map without model metadata.
;; can-write? requires: can-read?, has-db-transforms-permission?, and transforms-editable?
;; can-read? requires: is-superuser? OR (is-data-analyst? AND source-tables-readable?)
- (let [source-db-id (or (:source_database_id instance) (transforms.i/source-db-id instance))]
+ (let [source-db-id (or (:source_database_id instance) (transforms-base.i/source-db-id instance))]
(and (or api/*is-superuser?*
(and api/*is-data-analyst?*
- (transforms.util/source-tables-readable? instance)))
+ (transforms.u/source-tables-readable? instance)))
(perms/has-db-transforms-permission? api/*current-user-id* source-db-id)
(remote-sync/transforms-editable?))))
@@ -72,7 +73,10 @@
[source-tables]
(update-vals source-tables #(if (map? %) (update-keys % keyword) %)))
-(defn- transform-source-out [m]
+(defn transform-source-out
+ "Deserialize a transform source map from JSON storage format.
+ Normalizes queries, keywordizes types and source-table refs."
+ [m]
(-> m
mi/json-out-without-keywordization
(update-keys keyword)
@@ -81,9 +85,12 @@
(m/update-existing :type keyword)
(m/update-existing :source-incremental-strategy #(update-keys % keyword))))
-(defn- transform-source-in [m]
+(defn transform-source-in
+ "Serialize a transform source map for JSON storage.
+ Normalizes source-tables and prepares queries for serialization."
+ [m]
(-> m
- (m/update-existing :source-tables transforms.util/normalize-source-tables)
+ (m/update-existing :source-tables transforms-base.u/normalize-source-tables)
(m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))
mi/json-in))
@@ -102,7 +109,7 @@
(collection/check-collection-namespace :model/Transform collection_id)
(when collection_id
(collection/check-allowed-content :model/Transform collection_id))
- (let [target-db-id (transforms.i/target-db-id transform)
+ (let [target-db-id (transforms-base.i/target-db-id transform)
;; This is defensive code to cope with some tests for remote sync, where we deserialize a transform
;; with a concrete database id within it, for a potentially non-existent database.
;; In practice, our serialized representation should not contain any database ids, and this should
@@ -114,9 +121,9 @@
(-> transform
(assoc-in [:target :database] target-db-id)
(assoc
- :source_type (transforms.util/transform-source-type source)
+ :source_type (transforms-base.u/transform-source-type source)
:target_db_id (when valid-db-id? target-db-id)
- :source_database_id (or source_database_id (transforms.i/source-db-id transform))))))
+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform))))))
(t2/define-before-update :model/Transform
[{:keys [source source_database_id] :as transform}]
@@ -125,17 +132,17 @@
(collection/check-allowed-content :model/Transform new-collection))
(cond-> transform
source
- (assoc :source_type (transforms.util/transform-source-type source)
- :source_database_id (or source_database_id (transforms.i/source-db-id transform)))
+ (assoc :source_type (transforms-base.u/transform-source-type source)
+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))
(or (:source (t2/changes transform)) (:target (t2/changes transform)))
;; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.
- (assoc :target_db_id (transforms.i/target-db-id transform))))
+ (assoc :target_db_id (transforms-base.i/target-db-id transform))))
(t2/define-after-select :model/Transform
[{:keys [source] :as transform}]
(if source
- (assoc transform :source_type (transforms.util/transform-source-type source))
+ (assoc transform :source_type (transforms-base.u/transform-source-type source))
transform))
(methodical/defmethod t2/batched-hydrate [:model/TransformRun :transform]
@@ -295,7 +302,7 @@
"Fetch tables with their fields. The tables show up under the `:table` property."
[transforms]
(let [table-key-fn (fn [{:keys [target] :as transform}]
- [(transforms.i/target-db-id transform) (:schema target) (:name target)])
+ [(transforms-base.i/target-db-id transform) (:schema target) (:name target)])
table-keys (into #{} (map table-key-fn) transforms)
table-keys-with-schema (filter second table-keys)
table-keys-without-schema (keep (fn [[db-id schema table-name]]
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index ce6ca9432be..9dce8efb814 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -7,11 +7,12 @@
[metabase.models.transforms.transform-run]
[metabase.models.transforms.transform-run-cancelation]
[metabase.models.transforms.transform-tag]
+ [metabase.transforms-base.ordering]
+ [metabase.transforms-base.util]
[metabase.transforms.canceling]
[metabase.transforms.crud]
[metabase.transforms.execute]
[metabase.transforms.jobs]
- [metabase.transforms.ordering]
[metabase.transforms.schedule]
[metabase.transforms.settings]
[metabase.transforms.util]
@@ -21,14 +22,15 @@
(p/import-vars
[metabase.transforms.settings
transform-timeout]
- [metabase.transforms.util
- add-source-readable
+ [metabase.transforms-base.util
native-query-transform?
python-transform?
query-transform?
transform-source-database
transform-source-type
- transform-type
+ transform-type]
+ [metabase.transforms.util
+ add-source-readable
is-temp-transform-table?]
[metabase.transforms.crud
python-source-table-ref->table-id
@@ -47,7 +49,7 @@
cancel-run!]
[metabase.transforms.execute
execute!]
- [metabase.transforms.ordering
+ [metabase.transforms-base.ordering
transform-ordering
get-transform-cycle]
[metabase.transforms.jobs
diff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.clj
index 4da9e2ef79d..59f1438012d 100644
--- a/src/metabase/transforms/crud.clj
+++ b/src/metabase/transforms/crud.clj
@@ -13,9 +13,10 @@
[metabase.models.interface :as mi]
[metabase.models.transforms.transform :as transform.model]
[metabase.query-processor.compile :as qp.compile]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.ordering :as transforms.ordering]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.ordering :as transforms-base.ordering]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.log :as log]
@@ -31,7 +32,7 @@
We now supported table-ref as source but since FE is still expecting table-id we need to temporarily do this.
Should update FE to fully use table-ref"
[transform]
- (if (transforms.util/python-transform? transform)
+ (if (transforms-base.u/python-transform? transform)
(update-in transform [:source :source-tables]
(fn [source-tables]
(update-vals source-tables #(if (int? %) % (:table_id %)))))
@@ -40,9 +41,9 @@
(defn check-database-feature
"Check that the target database supports the required features for this transform."
[transform]
- (let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))
+ (let [database (api/check-400 (t2/select-one :model/Database (transforms-base.i/target-db-id transform))
(deferred-tru "The target database cannot be found."))
- features (transforms.util/required-database-features transform)]
+ features (transforms-base.u/required-database-features transform)]
(api/check-400 (not (:is_sample database))
(deferred-tru "Cannot run transforms on the sample database."))
(api/check-400 (not (:is_audit database))
@@ -55,13 +56,13 @@
(defn check-feature-enabled!
"Check that the premium features required for this transform type are enabled."
[transform]
- (api/check (transforms.util/check-feature-enabled transform)
+ (api/check (transforms.u/check-feature-enabled transform)
[402 (deferred-tru "Premium features required for this transform type are not enabled.")]))
(defn validate-transform-query!
"Validate that the transform query is valid. Throws a 400 error if validation fails."
[transform]
- (when-let [error (transforms.util/validate-transform-query transform)]
+ (when-let [error (transforms-base.u/validate-transform-query transform)]
(throw (ex-info (:error error)
(assoc error
:status-code 400)))))
@@ -107,7 +108,7 @@
be supported by all drivers or for all query types."
[driver database-id query]
(some->> (extract-all-columns-from-query driver database-id query)
- (filter (comp transforms.util/supported-incremental-filter-type? :base_type))
+ (filter (comp transforms-base.u/supported-incremental-filter-type? :base_type))
(mapv :name)))
(defn validate-incremental-column-type!
@@ -130,7 +131,7 @@
checkpoint-filter-unique-key
(let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)]
(api/check-400 column (deferred-tru "Checkpoint column not found in query."))
- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))
+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base-type column))
(deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."
(pr-str (:base-type column)))))
@@ -138,7 +139,7 @@
checkpoint-filter
(when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))]
(when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]
- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))
+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type column))
(deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."
checkpoint-filter
(pr-str (:base_type column)))))))))))
@@ -146,30 +147,30 @@
(defn get-transforms
"Get a list of transforms."
[& {:keys [last-run-start-time last-run-statuses tag-ids]}]
- (let [enabled-types (transforms.util/enabled-source-types-for-user)]
+ (let [enabled-types (transforms.u/enabled-source-types-for-user)]
(api/check-403 (seq enabled-types))
(let [transforms (t2/select :model/Transform {:where [:in :source_type enabled-types]
:order-by [[:id :asc]]})]
(->> (t2/hydrate transforms :last_run :transform_tag_ids :creator :owner)
(into []
- (comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last-run-start-time)
- (transforms.util/->status-filter-xf [:last_run :status] last-run-statuses)
- (transforms.util/->tag-filter-xf [:tag_ids] tag-ids)
- (map #(update % :last_run transforms.util/localize-run-timestamps))
+ (comp (transforms-base.u/->date-field-filter-xf [:last_run :start_time] last-run-start-time)
+ (transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)
+ (transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)
+ (map #(update % :last_run transforms-base.u/localize-run-timestamps))
(map python-source-table-ref->table-id)))
- transforms.util/add-source-readable))))
+ transforms.u/add-source-readable))))
(defn get-transform
"Get a specific transform."
[id]
(let [{:keys [target] :as transform} (api/read-check :model/Transform id)
- target-table (transforms.util/target-table (transforms.i/target-db-id transform) target :active true)]
+ target-table (transforms-base.u/target-table (transforms-base.i/target-db-id transform) target :active true)]
(-> transform
(t2/hydrate :last_run :transform_tag_ids :creator :owner)
- (u/update-some :last_run transforms.util/localize-run-timestamps)
+ (u/update-some :last_run transforms-base.u/localize-run-timestamps)
(assoc :table target-table)
python-source-table-ref->table-id
- transforms.util/add-source-readable)))
+ transforms.u/add-source-readable)))
(defn create-transform!
"Create new transform in the appdb.
@@ -177,7 +178,7 @@
([body]
(create-transform! body nil))
([body creator-id]
- (when (transforms.util/query-transform? body)
+ (when (transforms-base.u/query-transform? body)
(validate-transform-query! body))
(let [creator-id (or creator-id api/*current-user-id*)
transform (t2/with-transaction [_]
@@ -214,13 +215,13 @@
(check-feature-enabled! new)
(check-database-feature new)
(validate-incremental-column-type! new)
- (when (transforms.util/query-transform? old)
+ (when (transforms-base.u/query-transform? old)
(validate-transform-query! new)
- (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]
+ (when-let [{:keys [cycle-str]} (transforms-base.ordering/get-transform-cycle new)]
(throw (ex-info (str "Cyclic transform definitions detected: " cycle-str)
{:status-code 400}))))
(api/check (not (and (not= (target-fields old) (target-fields new))
- (transforms.util/target-table-exists? new)))
+ (transforms-base.u/target-table-exists? new)))
403
(deferred-tru "A table with that name already exists.")))
(t2/update! :model/Transform id (dissoc body :tag_ids))
@@ -231,7 +232,7 @@
(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
(-> transform
python-source-table-ref->table-id
- transforms.util/add-source-readable)))
+ transforms.u/add-source-readable)))
(defn delete-transform!
"Delete a transform and publish the delete event."
diff --git a/src/metabase/transforms/interface.clj b/src/metabase/transforms/interface.clj
index b6fb789d31a..f788462e90e 100644
--- a/src/metabase/transforms/interface.clj
+++ b/src/metabase/transforms/interface.clj
@@ -1,32 +1,21 @@
-(ns metabase.transforms.interface)
+(ns metabase.transforms.interface
+ "Interface for scheduled transform execution.
-(defn- transform->transform-type
- [transform]
- (-> transform :source :type keyword))
+ For core interface methods (source-db-id, target-db-id, table-dependencies, execute-base!),
+ use metabase.transforms-base.interface directly.
-(defmulti source-db-id
- "Return the ID of the source database for a given `transform`. The source database is where the data originates from
- before being transformed and written to the target destination.
-
- NOTE: Currently returns a single database ID, but we may want to change this to return multiple database IDs in the
- future since transforms can have multiple input sources."
- {:added "0.57.0" :arglists '([transform])}
- transform->transform-type)
-
-(defmulti target-db-id
- "Return the ID of the target database for a given `transform`. The target database is the destination where the
- transformed data will be written. Often this is the same database as the source database."
- {:added "0.57.0" :arglists '([transform])}
- transform->transform-type)
+ This namespace defines the scheduled-execution-specific `execute!` multimethod
+ which handles transform_run tracking."
+ (:require
+ [metabase.transforms-base.interface :as transforms-base.i]))
+;; Scheduled execution multimethod - dispatches on transform type
+;; This is separate from transforms-base.i/execute-base! which does NOT track transform_run
(defmulti execute!
- "Execute a transform operation. Runs the actual transformation process, which might involve running SQL queries,
- Python scripts, or other transformation logic depending on the transform type.
-
- This method blocks and may take significant time depending on the data volume. Implementations
- should handle errors gracefully and provide appropriate logging.
+ "Execute a transform operation with transform_run tracking.
- Returns nil (or a result that can be discarded).
+ This method creates transform_run rows, tracks status, and handles cancellation.
+ For base execution without database tracking, use [[metabase.transforms-base.interface/execute-base!]] instead.
Options:
- `:start-promise`
@@ -45,24 +34,4 @@
Do not use this directly. Use [[metabase.transforms.execute/execute!]] instead."
{:added "0.57.0" :arglists '([transform options])}
(fn [transform _options]
- (transform->transform-type transform)))
-
-(defmulti table-dependencies
- "Return a set of logical table dependencies of the transform, including indirect dependencies via cards.
- The transform execution system uses these dependencies to determine the correct order of execution
- and to detect circular dependencies.
-
- Each dependency is represented as one of:
- - `{:table <table-id>}`
- A dependency on a table that exists and has been synced.
- - `{:transform <transform-id>}`
- A dependency on a table that does not yet exist, but is known to be the target of another transform.
- Represents a 'placeholder' table (as we no table id / metadata) for the same purposes.
- - `{:table-ref {:database_id <db-id> :schema <schema> :table <name>}}`
- A dependency on a table by name, for cases where the table_id may not exist yet (e.g., when
- bulk importing transforms that depend on each other's outputs). The ordering system will
- resolve these to transforms by matching against target table definitions.
-
- An empty set indicates no dependencies."
- {:added "0.57.0" :arglists '([transform])}
- transform->transform-type)
+ (transforms-base.i/transform->transform-type transform)))
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index b00121f4198..d733db93600 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -11,11 +11,11 @@
[metabase.models.transforms.transform-run :as transform-run]
[metabase.revisions.core :as revisions]
[metabase.task.core :as task]
+ [metabase.transforms-base.ordering :as transforms-base.ordering]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.instrumentation :as transforms.instrumentation]
- [metabase.transforms.ordering :as transforms.ordering]
[metabase.transforms.settings :as transforms.settings]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms.util :as transforms.u]
[metabase.util :as u]
[metabase.util.i18n :as i18n]
[metabase.util.log :as log]
@@ -34,7 +34,7 @@
found)))
(defn- next-transform [ordering transforms-by-id complete]
- (-> (transforms.ordering/available-transforms ordering #{} complete)
+ (-> (transforms-base.ordering/available-transforms ordering #{} complete)
first
transforms-by-id))
@@ -52,7 +52,7 @@
(defn- get-plan [transform-ids]
(let [all-transforms (t2/select :model/Transform)
- global-ordering (transforms.ordering/transform-ordering all-transforms)
+ global-ordering (transforms-base.ordering/transform-ordering all-transforms)
relevant-ids (get-deps global-ordering transform-ids)
transforms-by-id (into {}
(keep (fn [{:keys [id] :as transform}]
@@ -60,7 +60,7 @@
[id transform])))
all-transforms)
ordering (sorted-ordering (select-keys global-ordering relevant-ids) transforms-by-id)]
- (when-let [cycle (transforms.ordering/find-cycle ordering)]
+ (when-let [cycle (transforms-base.ordering/find-cycle ordering)]
(let [id->name (into {} (map (juxt :id :name)) all-transforms)]
(throw (ex-info (str "Cyclic transform definitions detected: "
(str/join " → " (map id->name cycle)))
@@ -78,7 +78,7 @@
(Thread/sleep 2000))))
(defn- run-transform! [run-id run-method user-id {transform-id :id :as transform}]
- (if-not (transforms.util/check-feature-enabled transform)
+ (if-not (transforms.u/check-feature-enabled transform)
(log/warnf "Skip running transform %d due to lacking premium features" transform-id)
(do
(block-until-not-already-running transform-id)
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index e53845b5814..ee8c2751454 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -1,99 +1,57 @@
(ns metabase.transforms.query-impl
(:require
- [metabase.database-routing.core :as database-routing]
+ [clojure.core.async :as a]
[metabase.driver :as driver]
[metabase.driver.connection :as driver.conn]
- [metabase.driver.util :as driver.u]
- [metabase.lib.schema.common :as schema.common]
- [metabase.query-processor.compile :as qp.compile]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.interface :as transforms.i]
- [metabase.transforms.util :as transforms.util]
- [metabase.util.i18n :as i18n]
+ [metabase.transforms.util :as transforms.u]
[metabase.util.log :as log]
- [metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
-(defmethod transforms.i/source-db-id :query
- [transform]
- (-> transform :source :query :database))
-
-(defmethod transforms.i/target-db-id :query
- [transform]
- ;; For query transforms, the target needs to match the source, so use the query as the source of truth.
- (or (-> transform :source :query :database)
- ;; Fallback to using a configured value.
- (get-in transform [:target :database])
- (:target_db_id transform)))
-
-(mr/def ::transform-details
- [:map
- [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
- [:conn-spec :any]
- [:query ::qp.compile/compiled]
- [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
-
-(mr/def ::transform-opts
- [:map
- [:overwrite? :boolean]])
-
-(defn- transform-opts [_transform-details]
- ;; once we have more than just :table and :table-incremental as transform-types,
- ;; then we can dispatch on :target-incremental-strategy
- {})
-
-(defn- throw-if-db-routing-enabled [transform driver database]
- (when (database-routing/db-routing-enabled? database)
- (throw (ex-info (i18n/tru "Failed to run the transform ({0}) because the database ({1}) has database routing turned on. Running transforms on databases with db routing enabled is not supported."
- (:name transform)
- (:name database))
- {:driver driver, :database database}))))
-
(defn- run-mbql-transform!
([transform] (run-mbql-transform! transform nil))
- ([{:keys [id source target owner_user_id creator_id] :as transform} {:keys [run-method start-promise user-id]}]
+ ([{:keys [id source target owner_user_id creator_id] :as transform}
+ {:keys [run-method start-promise user-id]}]
(try
- (let [db (get-in source [:query :database])
- {driver :engine :as database} (t2/select-one :model/Database db)
- ;; important to test routing before calling compile-source (whose qp middleware will also throw)
- _ (throw-if-db-routing-enabled transform driver database)]
+ (let [db (t2/select-one :model/Database (get-in source [:query :database]))
+ driver (:engine db)
+ _ (transforms-base.u/throw-if-db-routing-enabled! transform db)
+ run-user-id (if (and (= run-method :manual) user-id)
+ user-id
+ (or owner_user_id creator_id))
+ {run-id :id} (transforms.u/try-start-unless-already-running id run-method run-user-id)]
+ (when start-promise (deliver start-promise [:started run-id]))
(driver.conn/with-write-connection
- (let [conn-spec (driver/connection-spec driver database)
- transform-details {:db-id db
- :database database
- :transform-id id
- :transform-type (keyword (:type target))
- :conn-spec conn-spec
- :query (transforms.util/compile-source transform)
- :output-schema (:schema target)
- :output-table (transforms.util/qualified-table-name driver target)}
- opts (transform-opts transform-details)
- features (transforms.util/required-database-features transform)
- ;; For manual runs, use the triggering user; for cron, use owner/creator
- run-user-id (if (and (= run-method :manual) user-id)
- user-id
- (or owner_user_id creator_id))]
- (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
- (throw (ex-info "The database does not support the requested transform target type."
- {:driver driver, :database database, :features features})))
- ;; mark the execution as started and notify any observers
- (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method run-user-id)]
- (when start-promise
- (deliver start-promise [:started run-id]))
- (log/info "Executing transform" id "with target" (pr-str target)
- (when (driver.conn/write-connection-requested?)
- " using write connection"))
- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
- (transforms.util/run-cancelable-transform!
- run-id driver transform-details
- (fn [_cancel-chan]
- (driver/run-transform! driver transform-details opts))))
- (transforms.util/handle-transform-complete!
- :run-id run-id
- :transform transform
- :db database)))))
+ (log/info "Executing transform" id "with target" (pr-str target)
+ (when (driver.conn/write-connection-requested?) " using write connection"))
+ (let [conn-spec (driver/connection-spec driver db)
+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}]
+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]
+ (transforms.u/run-cancelable-transform!
+ run-id driver transform-details
+ (fn [cancel-chan]
+ (let [result (transforms-base.i/execute-base!
+ transform
+ {:cancelled? #(boolean (a/poll! cancel-chan))
+ :run-id run-id
+ :with-stage-timing-fn (fn [rid stage thunk]
+ (transforms.instrumentation/with-stage-timing [rid stage]
+ (thunk)))})]
+ ;; Bridge result-map to exception-based flow for run-cancelable-transform!
+ (when-not (= :succeeded (:status result))
+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
+ result))))
+ ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)
+ (transforms-base.u/complete-execution! transform
+ {:run-id run-id
+ :with-stage-timing-fn (fn [rid stage thunk]
+ (transforms.instrumentation/with-stage-timing [rid stage]
+ (thunk)))}))))
(catch Throwable t
(if (= :already-running (:error (ex-data t)))
(log/warnf "Transform %d is already running" id)
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index f0c61c0d308..bdc495e2801 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -1,116 +1,38 @@
(ns metabase.transforms.util
+ "Transform utilities for scheduled execution.
+
+ Most pure utilities are in metabase.transforms-base.util and re-exported here.
+ This namespace adds scheduled-execution-specific functions that depend on
+ transform_run tracking, cancellation, instrumentation, and feature gating."
(:require
- [buddy.core.codecs :as codecs]
- [buddy.core.hash :as buddy-hash]
[clojure.core.async :as a]
[clojure.string :as str]
- [java-time.api :as t]
[metabase.api.common :as api]
[metabase.driver :as driver]
[metabase.driver.sql-jdbc :as sql-jdbc]
- [metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.util :as driver.u]
- [metabase.events.core :as events]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.schema.common :as lib.schema.common]
[metabase.models.interface :as mi]
[metabase.models.transforms.transform-run :as transform-run]
- [metabase.query-processor :as qp]
- [metabase.query-processor.compile :as qp.compile]
- [metabase.query-processor.middleware.add-remaps :as remap]
- [metabase.query-processor.middleware.catch-exceptions :as qp.catch-exceptions]
- [metabase.query-processor.parameters.dates :as params.dates]
[metabase.query-processor.pipeline :as qp.pipeline]
- [metabase.query-processor.preprocess :as qp.preprocess]
- [metabase.sync.core :as sync]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.canceling :as canceling]
[metabase.transforms.feature-gating :as transforms.gating]
- [metabase.transforms.instrumentation :as transforms.instrumentation]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.schema :as transforms.schema]
[metabase.transforms.settings :as transforms.settings]
[metabase.util :as u]
- [metabase.util.date-2 :as u.date]
- [metabase.util.honey-sql-2 :as h2x]
- [metabase.util.log :as log]
- [metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]
[toucan2.core :as t2])
(:import
- (java.sql SQLException)
- (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
- (java.util Date)))
+ (java.sql SQLException)))
(set! *warn-on-reflection* true)
-(defn qualified-table-name
- "Return the name of the target table of a transform as a possibly qualified symbol."
- [_driver {:keys [schema name]}]
- (if schema
- (keyword schema name)
- (keyword name)))
-
-(defn transform-type
- "Get the type of a transform"
- [transform]
- (-> transform :source :type keyword))
-
-(defn query-transform?
- "Check if this is a query transform: native query / mbql query."
- [transform]
- (= :query (transform-type transform)))
-
-(defn native-query-transform?
- "Check if this is a native query transform.
- Note: The transform should be normalized (via `normalize-transform`) before calling this function."
- [transform]
- (when (query-transform? transform)
- (let [query (-> transform :source :query)]
- (lib/native-only-query? query))))
-
-(defn python-transform?
- "Check if this is a Python transform."
- [transform]
- (= :python (transform-type transform)))
-
-(defn transform-source-database
- "Get the source database from a transform"
- [transform]
- (case (transform-type transform)
- :query (-> transform :source :query :database)
- :python (-> transform :source :source-database)))
-
-(defn normalize-transform
- "Normalize a transform's source query, similar to how transforms are normalized when read from the database.
- This should be called on transforms before processing them to ensure queries are in the expected format."
- [transform]
- (if (and (map? transform)
- (= (:type transform) "transform")
- (get-in transform [:source :query]))
- (update-in transform [:source :query] lib-be/normalize-query)
- transform))
-
-(defn transform-source-type
- "Returns the type of a transform's source: :python, :native, or :mbql.
- Throws if the source type cannot be detected.
- Note: The transform should be normalized (via `normalize-transform`) before calling this function."
- [source]
- (case (keyword (:type source))
- :python :python
- :query (if (lib/native-only-query? (:query source))
- :native
- :mbql)
- (throw (ex-info (str "Unknown transform source type: " (:type source))
- {:source source}))))
+;;; ------------------------------------------------- Feature Gating -------------------------------------------------
(defn check-feature-enabled
"Checking whether we have proper feature flags for using a given transform."
[transform]
(cond
- (query-transform? transform) (transforms.gating/query-transforms-enabled?)
- (python-transform? transform) (transforms.gating/python-transforms-enabled?)
+ (transforms-base.u/query-transform? transform) (transforms.gating/query-transforms-enabled?)
+ (transforms-base.u/python-transform? transform) (transforms.gating/python-transforms-enabled?)
:else false))
(defn enabled-source-types-for-user
@@ -161,6 +83,8 @@
transform-or-transforms)
(assoc transform-or-transforms :source_readable (source-tables-readable? transform-or-transforms))))
+;;; ------------------------------------------------- Scheduled Execution -------------------------------------------------
+
(defn- duplicate-key-violation?
"Check if an exception is a duplicate key violation.
Returns true for Postgres, MySQL/MariaDB, and H2 duplicate key errors."
@@ -204,554 +128,16 @@
(transform-run/succeed-started-run! run-id)
ret)
(catch Throwable t
- (transform-run/fail-started-run! run-id {:message (ex-message-fn t)})
+ (if (:timeout (ex-data t))
+ (transform-run/timeout-run! run-id {:message (ex-message-fn t)})
+ (transform-run/fail-started-run! run-id {:message (ex-message-fn t)}))
(throw t))
(finally
(canceling/chan-end-run! run-id))))
-(declare activate-table-and-mark-computed! target-table)
-
-(defn sync-target!
- "Sync target of a transform"
- [target database]
- ;; sync the new table (note that even a failed sync status means that the execution succeeded)
- (log/info "Syncing target" (pr-str target) "for transform")
- (activate-table-and-mark-computed! database target))
-
-(defn target-table-exists?
- "Test if the target table of a transform already exists."
- [{:keys [target] :as transform}]
- (let [db-id (transforms.i/target-db-id transform)
- {driver :engine :as database} (t2/select-one :model/Database db-id)]
- (driver/table-exists? driver database target)))
-
-(defn target-table
- "Load the `target` table of a transform from the database specified by `database-id`."
- [database-id target & kv-args]
- (some-> (apply t2/select-one :model/Table
- :db_id database-id
- :schema (:schema target)
- :name (:name target)
- kv-args)
- (t2/hydrate :db)))
-
-(defn- sync-table!
- ([database target] (sync-table! database target nil))
- ([database target {:keys [create?]}]
- (when-let [table (or (target-table (:id database) target)
- (when create?
- (sync/create-table! database (select-keys target [:schema :name :data_source :data_authority]))))]
- (sync/sync-table! table)
- table)))
-
-(defn activate-table-and-mark-computed!
- "Activate table for `target` in `database` in the app db."
- [database target]
- (when-let [table (sync-table! database (assoc target
- :data_authority :computed
- :data_source :metabase-transform)
- {:create? true})]
- (when-not (:active table)
- (t2/update! :model/Table (:id table) {:active true}))
- table))
-
-(defn deactivate-table!
- "Deactivate table for `target` in `database` in the app db."
- [database target]
- (when-let [table (sync-table! database target)]
- ;; TODO this should probably be a function in the sync module
- (t2/update! :model/Table (:id table) {:active false})))
-
-(defn delete-target-table!
- "Delete the target table of a transform and sync it from the app db."
- [{:keys [id target], :as transform}]
- (when target
- (let [target (update target :type keyword)
- database-id (transforms.i/target-db-id transform)]
- (when database-id
- (if-let [{driver :engine :as database} (t2/select-one :model/Database database-id)]
- (do
- (driver/drop-transform-target! driver database target)
- (log/info "Deactivating target " (pr-str target) "for transform" id)
- (deactivate-table! database target))
- (log/warnf "Skipping drop of transform target %s for transform %d: database %d not found"
- (pr-str target) id database-id))))))
-
-(defn delete-target-table-by-id!
- "Delete the target table of the transform specified by `transform-id`."
- [transform-id]
- (delete-target-table! (t2/select-one :model/Transform transform-id)))
-
-(defn massage-sql-query
- "Adjusts mbql query for use in a transform."
- [query]
- (-> query
- remap/disable-remaps
- lib/disable-default-limit))
-
-(defn- checkpoint-incremental?
- "Returns true if `source` uses checkpoint-based incremental strategy."
- [source]
- (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))
-
-(defn supported-incremental-filter-type?
- "Returns true if the given base-type is supported for incremental filtering.
-
- We only support temporal (timestamp/tz) and numeric (int/float) types."
- [base-type]
- (or (isa? base-type :type/Temporal)
- (isa? base-type :type/Number)))
-
-(defn- source->checkpoint-filter-unique-key
- "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."
- [query source-incremental-strategy]
- (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))
-
-(defn- source->checkpoint-filter-column
- "Resolve the checkpoint filter column for an incremental transform.
-
- Tries to resolve the column using the unique key first.
- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.
-
- Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).
- Throws an exception if the column type is not supported."
- [query source-incremental-strategy table metadata-provider]
- (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]
- (when-some [{column-name :name
- :keys [base-type]
- :as column}
- (cond
- checkpoint-filter-unique-key
- (source->checkpoint-filter-unique-key query source-incremental-strategy)
- checkpoint-filter
- (when-some [field-id (t2/select-one-pk :model/Field
- :table_id (:id table)
- :name checkpoint-filter)]
- (lib.metadata/field metadata-provider field-id)))]
- (when-not (supported-incremental-filter-type? base-type)
- (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "
- "Only numeric and temporal columns are supported for incremental filtering.")
- {:column-name column-name
- :base-type base-type})))
- column)))
-
-(defn next-checkpoint
- "Build a query to compute the MAX of the checkpoint column from the target table.
-
- Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),
- or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."
- [{:keys [source target] :as transform}]
- (let [db-id (transforms.i/target-db-id transform)]
- (when (checkpoint-incremental? source)
- (when-let [table (target-table db-id target)]
- (let [metadata-provider (lib-be/application-database-metadata-provider db-id)
- table-metadata (lib.metadata/table metadata-provider (:id table))
- query (lib/query metadata-provider table-metadata)]
- (when-let [filter-column (source->checkpoint-filter-column query
- (:source-incremental-strategy source)
- table metadata-provider)]
- {:query (-> query (lib/aggregate (lib/max filter-column)))
- :filter-column filter-column}))))))
-
-(defn- next-checkpoint-value
- "Execute the checkpoint query and normalize the result for database insertion.
- Returns `nil` if the target table is empty."
- [{:keys [query filter-column]}]
- (let [{:keys [base-type]} filter-column
- v (some-> query qp/process-query :data :rows first first)]
- ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right
- ;; shape for reinsertion
- (cond
- (nil? v)
- nil
-
- (isa? base-type :type/Integer)
- (bigint v)
-
- ;; any other number that's not an integer, should be a decimal/float
- (number? v)
- (bigdec v)
-
- :else v)))
-
-(defn preprocess-incremental-query
- "Add checkpoint checkpoint filtering to a query for incremental execution.
-
- For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.
- For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.
- Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."
- [query source-incremental-strategy checkpoint]
- (if-let [checkpoint-value (next-checkpoint-value checkpoint)]
- (if (lib/native? query)
- ;; native query with explicit checkpoint filter
- (if (get-in query [:stages 0 :template-tags "checkpoint"])
- (update query :parameters conj
- {:type (if (number? checkpoint-value) :number :text)
- :target [:variable [:template-tag "checkpoint"]]
- :value checkpoint-value})
- query)
- ;; mbql query
- (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))
- query))
-
-(defn- post-process-incremental-query
- "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.
-
- Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "
- [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]
- (let [{:keys [checkpoint-filter]} source-incremental-strategy]
- (if (and (lib/native? (:query source))
- (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))
- (next-checkpoint-value checkpoint))
- (let [wrap-query (fn [query]
- (let [honeysql-query {:select [:*]
- :from [[[:raw (str "(" query ")")] :subquery]]
- :where [:> (h2x/identifier :field checkpoint-filter)
- [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]
- (first (sql.qp/format-honeysql driver honeysql-query))))]
- (update outer-query :query wrap-query))
- outer-query)))
-
-(mu/defn validate-transform-query :- [:maybe [:map [:error :string]]]
- "Verifies that a query transform's query can actually be run as is. Returns nil on success and an error map on failure."
- [{:keys [source]}]
- (case (keyword (:type source))
- :query
- (try
- (qp.preprocess/preprocess (:query source))
- nil
- (catch Exception e
- (qp.catch-exceptions/exception-response e)))))
-
-(defn compile-source
- "Compile the source query of a transform to SQL, applying incremental filtering if required."
- [{:keys [source] :as transform}]
- (let [{:keys [source-incremental-strategy] query-type :type} source]
- (case (keyword query-type)
- :query
- (let [checkpoint (next-checkpoint transform)
- query (:query source)
- driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]
- (binding [driver/*compile-with-inline-parameters*
- (or (= :clickhouse driver)
- driver/*compile-with-inline-parameters*)]
- (-> query
- (preprocess-incremental-query source-incremental-strategy checkpoint)
- massage-sql-query
- qp.compile/compile
- (post-process-incremental-query driver source checkpoint)))))))
-
-(defn required-database-features
- "Returns the database features necessary to execute `transform`."
- [transform]
- (if (python-transform? transform)
- [:transforms/python]
- [:transforms/table]))
-
-(defn ->instant
- "Convert a temporal value `t` to an Instant in the system timezone."
- ^Instant [t]
- (when t
- (condp instance? t
- Instant t
- Date (.toInstant ^Date t)
- OffsetDateTime (.toInstant ^OffsetDateTime t)
- ZonedDateTime (.toInstant ^ZonedDateTime t)
- LocalDateTime (recur (.atZone ^LocalDateTime t (t/zone-id)))
- String (recur (u.date/parse t))
- LocalTime (recur (.atDate ^LocalTime t (t/local-date)))
- OffsetTime (recur (.atDate ^OffsetTime t (t/local-date)))
- LocalDate (recur (.atStartOfDay ^LocalDate t))
- (throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")
- {:temporal t})))))
-
-(defn utc-timestamp-string
- "Convert the timestamp t to a string encoding the it in the system timezone."
- [t]
- (-> t ->instant str))
-
-(defn localize-run-timestamps
- "Convert the timestamps of a `run` to ISO strings in UTC."
- [run]
- (-> run
- (u/update-some :start_time utc-timestamp-string)
- (u/update-some :end_time utc-timestamp-string)))
-
-(mr/def ::column-definition
- [:map
- [:name :string]
- [:type ::lib.schema.common/base-type]
- [:nullable? {:optional true} :boolean]])
-
-(mr/def ::table-definition
- [:map
- [:name :keyword]
- [:columns [:sequential ::column-definition]]
- [:primary-key {:optional true} [:sequential :string]]])
-
-(mu/defn create-table-from-schema!
- "Create a table from a table-schema"
- [driver :- :keyword
- database-id :- pos-int?
- table-schema :- ::table-definition]
- (let [{:keys [columns] table-name :name} table-schema
- column-definitions (mapv (fn [{:keys [name type database-type]}]
- (let [db-type (if database-type
- [[:raw database-type]]
- (try
- (driver/type->database-type driver type)
- (catch IllegalArgumentException _
- (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)
- (driver/type->database-type driver :type/Text))))]
- [name db-type]))
- columns)
- primary-key-opts (select-keys table-schema [:primary-key])]
- (log/infof "Creating table %s with %d columns" table-name (count columns))
- (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
-
-(defn drop-table!
- "Drop a table in the database."
- [driver database-id table-name]
- (log/infof "Dropping table %s" table-name)
- (driver/drop-table! driver database-id table-name))
-
-(defn rename-tables!
- "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.
- This is a simpler, composable operation that only handles renaming."
- [driver database-id rename-map]
- (log/infof "Renaming tables: %s" (pr-str rename-map))
- (driver/rename-tables! driver database-id rename-map))
-
(defn is-temp-transform-table?
"Return true when `table` matches the transform temporary table naming pattern and transforms are enabled."
[table]
(boolean
(when-let [table-name (and (transforms.gating/any-transforms-enabled?) (:name table))]
(str/starts-with? (u/lower-case-en table-name) driver.u/transform-temp-table-prefix))))
-
-;;; ------------------------------------------------- Source Table Resolution -----------------------------------------
-
-(def ^:private ^:const batch-lookup-chunk-size
- "Maximum number of table refs to query at once to avoid SQL query size limits."
- 100)
-
-(defn batch-lookup-table-ids
- "Batch lookup table IDs from ref maps. Returns {[db_id schema name] -> table_id}.
- Queries the exact conjunction of each [database_id, schema, table] triple rather than
- a Cartesian product of all values. Uses chunking to avoid query size limits."
- [refs]
- (when (seq refs)
- (let [unique-refs (distinct (map (juxt :database_id :schema :table) refs))
- ref->clause (fn [[db-id schema table-name]]
- [:and
- [:= :db_id db-id]
- (if (some? schema)
- [:= :schema schema]
- [:is :schema nil])
- [:= :name table-name]])
- fetch-batch (fn [batch]
- (t2/select-fn->fn (juxt :db_id :schema :name) :id
- [:model/Table :id :db_id :schema :name]
- {:where (into [:or] (map ref->clause batch))}))]
- (into {} (mapcat fetch-batch) (partition-all batch-lookup-chunk-size unique-refs)))))
-
-(defn- source-table-ref->key
- "Convert a source table ref map to a lookup key [db_id schema name]."
- [{:keys [database_id schema table]}]
- [database_id schema table])
-
-(defn- missing-table-id?
- "Returns true if `v` is a source table ref map that needs table_id lookup."
- [v]
- (and (map? v) (nil? (:table_id v))))
-
-(defn normalize-source-tables
- "Normalize source-tables to consistent map format {:database_id :schema :table :table_id}.
-
- The old format stored just integer table IDs. New transforms store maps on write.
- Old data is converted on read via transform-source-out for backwards compatibility.
-
- Throws if an integer table ID references a non-existent table.
- Map refs with non-existent tables get nil table_id (resolved later at execute time)."
- [source-tables]
- (let [int-table-ids (into #{} (filter int?) (vals source-tables))
- int-id->metadata (when (seq int-table-ids)
- (t2/select-pk->fn (fn [{:keys [db_id schema name]}]
- {:database_id db_id :schema schema :table name})
- [:model/Table :id :db_id :schema :name]
- :id [:in int-table-ids]))
- missing-ids (when (seq int-table-ids)
- (remove int-id->metadata int-table-ids))
- refs-needing-id (filter missing-table-id? (vals source-tables))
- ref-lookup (or (batch-lookup-table-ids refs-needing-id) {})]
- (when (seq missing-ids)
- (throw (ex-info (str "Tables not found for ids: " (str/join ", " (sort missing-ids)))
- {:table_ids (vec missing-ids)})))
- (update-vals source-tables
- (fn [v]
- (cond
- (int? v) (assoc (int-id->metadata v) :table_id v)
- (:table_id v) v
- :else (assoc v :table_id (ref-lookup (source-table-ref->key v))))))))
-
-(defn resolve-source-tables
- "Resolve source-tables to {alias -> table_id}. Throws if any table not found.
- For execute time - all entries must resolve to valid table IDs.
- Handles both integer IDs (old format) and map refs (new format)."
- [source-tables]
- (let [needs-lookup (filter missing-table-id? (vals source-tables))
- lookup (or (batch-lookup-table-ids needs-lookup) {})
- resolved (u/for-map [[alias v] source-tables]
- [alias (if (int? v)
- v
- (or (:table_id v) (lookup (source-table-ref->key v))))])
- unresolved (for [[alias table-id] resolved
- :when (nil? table-id)
- :let [v (get source-tables alias)]]
- {:alias alias
- :table (if-let [schema (:schema v)]
- (str schema "." (:table v))
- (:table v))
- :ref v})]
- (when (seq unresolved)
- (throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved)))
- {:unresolved unresolved
- :transform-message "Input table not found"})))
- resolved))
-
-(defn- matching-timestamp?
- [job field-path {:keys [start end]}]
- (when-let [field-instant (->instant (get-in job field-path))]
- (let [parse #(-> % u.date/parse ->instant)]
- ;; logic here is to find when it's not matching and invert this
- (not (or (and start (.isBefore field-instant (parse start)))
- (and end (.isAfter field-instant (parse end))))))))
-
-(defn ->date-field-filter-xf
- "Returns an xform for a date filter."
- [field-path filter-value]
- (if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
- (filter #(matching-timestamp? % field-path range))
- identity))
-
-(defn ->status-filter-xf
- "Returns an xform for a transform run status filter."
- [field-path statuses]
- (if-let [statuses (->> statuses (map keyword) set not-empty)]
- (filter #(statuses (get-in % field-path)))
- identity))
-
-(defn ->tag-filter-xf
- "Returns an xform for a transform tag filter."
- [field-path tag-ids]
- (if-let [tag-ids (-> tag-ids set not-empty)]
- (filter #(some tag-ids (get-in % field-path)))
- identity))
-
-(def ^:private metabase-index-prefix "mb_transform_idx_")
-
-(defn- incremental-filter-index-name [schema table-name filter-column-name]
- (let [prefix metabase-index-prefix
- suffix (codecs/bytes->hex (buddy-hash/md5 (str/join "|" [schema table-name filter-column-name])))]
- (str prefix suffix)))
-
-(defn- should-index-incremental-filter-column? [driver database]
- (and (driver.u/supports? driver :describe-indexes database)
- (driver.u/supports? driver :transforms/index-ddl database)))
-
-(defn- metabase-owned-index? [index]
- (str/starts-with? (:index-name index) metabase-index-prefix))
-
-(defn- metabase-incremental-filter-index [filter-column target]
- (let [table-name (:name target)
- schema (:schema target)
- filter-name (:name filter-column)]
- ;; match the schema used by describe-table-indexes (:value denotes the column name, assumed leading column if a composite index)
- {:index-name (incremental-filter-index-name schema table-name filter-name)
- :value filter-name}))
-
-(defn- decide-secondary-index-ddl
- "Decides which indexes should be dropped/created on the target table.
-
- e.g. Indexing the incremental :filter-column to accelerate MAX(target.checkpoint) queries on OLTP databases.
-
- Indexes are represented as maps with at least the keys :index-name, :value.
- This matches the form used by [[metabase.driver/describe-table-indexes]].
-
- Returns a map:
- :drop - a vector of indexes that are redundant and should be dropped.
- :create - a vector of desirable indexes that do not exist and that should be created.
-
- Notes:
- - If user covering indexes exist they should be reused.
- - Will never drop user indexes.
- - Indexes we previously created and are no longer required are dropped."
- [{:keys [filter-column indexes target database]}]
- (let [[mb-indexes user-indexes] ((juxt filter remove) metabase-owned-index? indexes)
- default-mb-index (when filter-column (metabase-incremental-filter-index filter-column target))
- column-name (:name filter-column)
- existing-user-index (first (filter #(= (:value %) column-name) user-indexes))
- existing-mb-index (first (filter #(= (:index-name %) (:index-name default-mb-index)) mb-indexes))
- driver (:engine database)
- intended-index (when (should-index-incremental-filter-column? driver database)
- ;; Prefer reuse to not create redundant indexes
- (or existing-user-index default-mb-index))
- drop (if intended-index
- (remove #(= (:index-name intended-index) (:index-name %)) mb-indexes)
- mb-indexes)
- create (when (and intended-index
- (not= (:index-name intended-index) (:index-name existing-user-index))
- (not= (:index-name intended-index) (:index-name existing-mb-index)))
- [intended-index])]
- {:drop (vec drop)
- :create (vec create)}))
-
-(defn execute-secondary-index-ddl-if-required!
- "If target table index modifications are required, executes those CREATE/DROP commands.
- See [[metabase.transforms-util/decide-secondary-index-ddl]] for details."
- [transform run-id database target]
- (when (driver.u/supports? (:engine database) :describe-indexes database)
- (let [driver (:engine database)
- indexes (driver/describe-table-indexes driver database target)
- checkpoint (next-checkpoint transform)
- {:keys [drop create]}
- (decide-secondary-index-ddl
- {:filter-column (:filter-column checkpoint)
- :database database
- :target target
- :indexes indexes})]
- (doseq [{:keys [index-name value]} drop]
- (transforms.instrumentation/with-stage-timing [run-id [:import :drop-incremental-filter-index]]
- (log/infof "Dropping secondary index %s(%s) for target %s" index-name value (pr-str target))
- (driver/drop-index! driver (:id database) (:schema target) (:name target) index-name)))
- (doseq [{:keys [index-name value]} create]
- (transforms.instrumentation/with-stage-timing [run-id [:import :create-incremental-filter-index]]
- (log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))
- (driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))
-
-(mu/defn handle-transform-complete!
- "Handles followup tasks for when a transform has completed.
-
- Specifically, this syncs the target db, publishes a `:event/transform-run-complete` event, and potentially updates
- the target table's index.
-
- See [[metabase.transforms-util/decide-secondary-index-ddl]] for details on the index handling."
- [& {:keys [run-id transform db]}
- :- [:map
- [:run-id ::transforms.schema/run-id]
- [:transform ::transforms.schema/transform]
- [:db [:fn {:error/message "Must a t2 database object"} #(= (t2/model %) :model/Database)]]]]
- (let [target (:target transform)]
- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]
- (when-let [table (sync-target! target db)]
- (t2/update! :model/Table (:id table) {:transform_id (:id transform)}))
- ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (:id db)
- :transform-id (:id transform)
- :transform-type (keyword (:type target))
- :output-schema (:schema target)
- :output-table (qualified-table-name (:engine db) target)}})
- ;; Creating an index after sync means the filter column is known in the appdb.
- ;; The index would be synced the next time sync runs, but at time of writing, index sync is disabled.
- (execute-secondary-index-ddl-if-required! transform run-id db target))))
diff --git a/src/metabase/transforms_base/core.clj b/src/metabase/transforms_base/core.clj
new file mode 100644
index 00000000000..df3f8e654d7
--- /dev/null
+++ b/src/metabase/transforms_base/core.clj
@@ -0,0 +1,46 @@
+(ns metabase.transforms-base.core
+ "Entry point for base transform execution.
+
+ This namespace provides the main `execute!` function for running transforms
+ without writing to transform_run rows. Results are returned in memory.
+
+ For query transforms, this module handles execution directly.
+ For python transforms, implementations are loaded separately via
+ metabase-enterprise.transforms-python.base to avoid circular dependencies."
+ (:require
+ ;; Interface for dispatch
+ [metabase.transforms-base.interface :as transforms-base.i]
+ ;; Load query implementation - registers multimethods
+ [metabase.transforms-base.query]
+ [metabase.transforms-base.util :as transforms-base.u]))
+
+(defn execute!
+ "Execute transform and return result map without writing transform_run rows.
+
+ This is the main entry point for base transform execution. Use this when you
+ want to run a transform without database tracking (e.g., from workspaces).
+
+ Performs the core execution via `execute-base!`, then runs post-processing
+ (sync, events, secondary indexes) via `complete-execution!`.
+
+ Options:
+ - `:cancelled?` - (fn [] boolean), polled during execution to check for cancellation
+ - `:run-id` - optional, for instrumentation/metrics (nil skips metrics recording)
+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk] result)
+ - `:publish-events?` - whether to publish Metabase events (default true)
+
+ Returns a map:
+ {:status :succeeded | :failed | :cancelled | :timeout
+ :result <implementation-specific result>
+ :error <exception if failed>
+ :logs <string, for python transforms>}
+
+ Target table sync (metabase_table writes) still occurs - this is necessary
+ for the table to be usable after execution."
+ ([transform]
+ (execute! transform nil))
+ ([transform opts]
+ (let [result (transforms-base.i/execute-base! transform opts)]
+ (when (= :succeeded (:status result))
+ (transforms-base.u/complete-execution! transform opts))
+ result)))
diff --git a/src/metabase/transforms_base/init.clj b/src/metabase/transforms_base/init.clj
new file mode 100644
index 00000000000..f5b5087792d
--- /dev/null
+++ b/src/metabase/transforms_base/init.clj
@@ -0,0 +1,9 @@
+(ns metabase.transforms-base.init
+ "Loads multimethod implementations for the transforms-base module.
+
+ `:query` implementations are loaded here directly.
+ `:python` implementations are loaded via the enterprise init chain
+ (metabase-enterprise.transforms-python.init)."
+ (:require
+ [metabase.transforms-base.ordering]
+ [metabase.transforms-base.query]))
diff --git a/src/metabase/transforms_base/interface.clj b/src/metabase/transforms_base/interface.clj
new file mode 100644
index 00000000000..2efddae8174
--- /dev/null
+++ b/src/metabase/transforms_base/interface.clj
@@ -0,0 +1,76 @@
+(ns metabase.transforms-base.interface
+ "Interface definitions for base transform execution.
+
+ This namespace defines the multimethods that different transform types
+ (query, python) must implement. The base module uses these for dispatch
+ without any database writes for transform_run tracking.
+
+ For scheduled execution with transform_run tracking, see metabase.transforms.interface.")
+
+(defn transform->transform-type
+ "Extract the transform type from a transform's source."
+ [transform]
+ (-> transform :source :type keyword))
+
+(defmulti source-db-id
+ "Return the ID of the source database for a given `transform`. The source database is where the data originates from
+ before being transformed and written to the target destination.
+
+ NOTE: Currently returns a single database ID, but we may want to change this to return multiple database IDs in the
+ future since transforms can have multiple input sources."
+ {:added "0.57.0" :arglists '([transform])}
+ transform->transform-type)
+
+(defmulti target-db-id
+ "Return the ID of the target database for a given `transform`. The target database is the destination where the
+ transformed data will be written. Often this is the same database as the source database."
+ {:added "0.57.0" :arglists '([transform])}
+ transform->transform-type)
+
+(defmulti table-dependencies
+ "Return a set of logical table dependencies of the transform, including indirect dependencies via cards.
+ The transform execution system uses these dependencies to determine the correct order of execution
+ and to detect circular dependencies.
+
+ Each dependency is represented as one of:
+ - `{:table <table-id>}`
+ A dependency on a table that exists and has been synced.
+ - `{:transform <transform-id>}`
+ A dependency on a table that does not yet exist, but is known to be the target of another transform.
+ Represents a 'placeholder' table (as we no table id / metadata) for the same purposes.
+ - `{:table-ref {:database_id <db-id> :schema <schema> :table <name>}}`
+ A dependency on a table by name, for cases where the table_id may not exist yet (e.g., when
+ bulk importing transforms that depend on each other's outputs). The ordering system will
+ resolve these to transforms by matching against target table definitions.
+
+ An empty set indicates no dependencies."
+ {:added "0.57.0" :arglists '([transform])}
+ transform->transform-type)
+
+(defmulti execute-base!
+ "Execute a transform's core logic and return results.
+
+ Does NOT:
+ - Write transform_run rows
+ - Sync target tables (caller should use `complete-execution!` or `transforms-base.core/execute!`)
+ - Publish events (caller should use `complete-execution!` or `transforms-base.core/execute!`)
+
+ Options:
+ - `:cancelled?` - (fn [] boolean), polled during execution to check for cancellation
+ - `:run-id` - optional, for instrumentation/metrics (nil skips metrics recording)
+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk]) for timing instrumentation
+ - `:message-log` - optional, pre-created message log atom (for python transforms)
+
+ Returns a map:
+ {:status :succeeded | :failed | :cancelled | :timeout
+ :result <implementation-specific result>
+ :error <exception if failed>
+ :logs <string, for python transforms>}
+
+ Implementations should:
+ - Check `(cancelled?)` at safe points and return {:status :cancelled} if true
+ - NOT create/update transform_run rows
+ - NOT sync target tables or publish events (caller handles via complete-execution!)"
+ {:added "0.57.0" :arglists '([transform options])}
+ (fn [transform _options]
+ (transform->transform-type transform)))
diff --git a/src/metabase/transforms/ordering.clj b/src/metabase/transforms_base/ordering.clj
similarity index 83%
rename from src/metabase/transforms/ordering.clj
rename to src/metabase/transforms_base/ordering.clj
index 654a1cb4a08..64b6998b614 100644
--- a/src/metabase/transforms/ordering.clj
+++ b/src/metabase/transforms_base/ordering.clj
@@ -1,4 +1,7 @@
-(ns metabase.transforms.ordering
+(ns metabase.transforms-base.ordering
+ "Transform dependency ordering and cycle detection.
+
+ Pure functions for computing execution order based on table dependencies."
(:require
[clojure.set :as set]
[clojure.string :as str]
@@ -10,36 +13,46 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.query-processor.preprocess :as qp.preprocess]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util.i18n :as i18n]
[metabase.util.malli :as mu]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (clojure.lang ExceptionInfo)))
(set! *warn-on-reflection* true)
+;;; ------------------------------------------------- Query Dependencies -------------------------------------------------
+
+(defn query-table-dependencies
+ "Compute table dependencies for a query transform.
+ This is the base implementation - callers may wrap with additional error handling."
+ [{:keys [source]}]
+ (let [query (-> (:query source)
+ transforms-base.u/massage-sql-query
+ qp.preprocess/preprocess)
+ driver (-> query
+ lib.metadata/database
+ :engine)]
+ (if (lib/native-only-query? query)
+ (driver/native-query-deps driver query)
+ (into #{}
+ (map (fn [table-id]
+ {:table table-id}))
+ (lib/all-source-table-ids query)))))
+
(defn- database-routing-error-ex-data [^Throwable e]
(when e
(if (:database-routing-enabled (ex-data e))
(ex-data e)
(recur (.getCause e)))))
-(defmethod transforms.i/table-dependencies :query
- [{:keys [source] :as transform}]
+(defmethod transforms-base.i/table-dependencies :query
+ [transform]
(try
- (let [query (-> (:query source)
- transforms.util/massage-sql-query
- qp.preprocess/preprocess)
- driver (-> query
- lib.metadata/database
- :engine)]
- (if (lib/native-only-query? query)
- (driver/native-query-deps driver query)
- (into #{}
- (map (fn [table-id]
- {:table table-id}))
- (lib/all-source-table-ids query))))
- (catch clojure.lang.ExceptionInfo e
+ (query-table-dependencies transform)
+ (catch ExceptionInfo e
(if-some [data (database-routing-error-ex-data e)]
(let [message (i18n/trs "Failed to run transform because the database {0} has database routing turned on. Running transforms on databases with db routing enabled is not supported." (:database-name data))]
(throw (ex-info message
@@ -49,9 +62,11 @@
e)))
(throw e)))))
+;;; ------------------------------------------------- Ordering Logic -------------------------------------------------
+
(defn- dependency-map [transforms]
(into {}
- (map (juxt :id transforms.i/table-dependencies))
+ (map (juxt :id transforms-base.i/table-dependencies))
transforms))
(mu/defn- output-table-map
@@ -95,7 +110,7 @@
(let [;; Group all transforms by their database, skipping transforms with no target db
transforms-by-db (->> transforms
(keep (fn [transform]
- (when-let [db-id (transforms.i/target-db-id transform)]
+ (when-let [db-id (transforms-base.i/target-db-id transform)]
{db-id [transform]})))
(apply merge-with into))
transform-ids (into #{} (map :id) transforms)
@@ -167,7 +182,7 @@
output-tables (output-table-map mp db-transforms)
transform-ids (into #{} (map :id) db-transforms)
target-refs (target-ref-map transforms)
- node->children #(->> % transforms-by-id transforms.i/table-dependencies
+ node->children #(->> % transforms-by-id transforms-base.i/table-dependencies
(keep (fn [dep] (resolve-dependency dep output-tables transform-ids target-refs))))
id->name (comp :name transforms-by-id)
cycle (find-cycle node->children [transform-id])]
diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.clj
new file mode 100644
index 00000000000..f00c8fd41b9
--- /dev/null
+++ b/src/metabase/transforms_base/query.clj
@@ -0,0 +1,138 @@
+(ns metabase.transforms-base.query
+ "Base query transform execution - core logic without transform_run tracking.
+
+ This namespace handles MBQL/native query transform execution and returns
+ results in memory rather than writing to transform_run rows."
+ (:require
+ [metabase.driver :as driver]
+ [metabase.driver.util :as driver.u]
+ [metabase.lib.schema.common :as schema.common]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.util.log :as log]
+ [metabase.util.malli.registry :as mr]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------- Interface Implementations -------------------------------------------------
+
+(defmethod transforms-base.i/source-db-id :query
+ [transform]
+ (-> transform :source :query :database))
+
+(defmethod transforms-base.i/target-db-id :query
+ [transform]
+ ;; For query transforms, the target needs to match the source, so use the query as the source of truth.
+ (or (-> transform :source :query :database)
+ ;; Fallback to using a configured value.
+ (get-in transform [:target :database])
+ (:target_db_id transform)))
+
+;;; ------------------------------------------------- Schemas -------------------------------------------------
+
+(mr/def ::transform-details
+ [:map
+ [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]
+ [:conn-spec :any]
+ [:query ::qp.compile/compiled]
+ [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])
+
+(mr/def ::transform-opts
+ [:map
+ [:overwrite? :boolean]])
+
+;;; ------------------------------------------------- Helpers -------------------------------------------------
+
+(defn- transform-opts [{:keys [transform-type]}]
+ (case transform-type
+ :table {:overwrite? true}
+ ;; once we have more than just append, dispatch on :target-incremental-strategy
+ :table-incremental {}))
+
+;;; ------------------------------------------------- Base Execution -------------------------------------------------
+
+(defn run-query-transform!
+ "Execute query transform (MBQL/native). Returns result map.
+
+ Does:
+ - Compile query
+ - Create schema if needed
+ - Call driver/run-transform!
+
+ Does NOT:
+ - Create transform_run row
+ - Update transform_run status
+ - Sync target table (caller handles via complete-execution!)
+ - Publish events (caller handles via complete-execution!)
+
+ Options:
+ - `:cancelled?` - (fn [] boolean), polled to check for cancellation
+ - `:run-id` - optional, for instrumentation (nil skips metrics)
+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk] result)
+
+ Returns:
+ {:status :succeeded | :failed | :cancelled
+ :result <driver result>
+ :error <exception if failed>}"
+ [{:keys [id source target] :as transform}
+ {:keys [cancelled?] :as _opts}]
+ (try
+ ;; Check cancellation before starting
+ (when (and cancelled? (cancelled?))
+ (throw (ex-info "Transform cancelled before start" {:status :cancelled})))
+
+ (let [db (get-in source [:query :database])
+ {driver :engine :as database} (t2/select-one :model/Database db)
+ _ (transforms-base.u/throw-if-db-routing-enabled! transform database)
+ transform-details {:db-id db
+ :database database
+ :transform-id id
+ :transform-type (keyword (:type target))
+ :conn-spec (driver/connection-spec driver database)
+ :query (transforms-base.u/compile-source transform)
+ :output-schema (:schema target)
+ :output-table (transforms-base.u/qualified-table-name driver target)}
+ opts (transform-opts transform-details)
+ features (transforms-base.u/required-database-features transform)]
+
+ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)
+ (throw (ex-info "The database does not support the requested transform target type."
+ {:driver driver, :database database, :features features})))
+
+ (log/info "Executing transform" id "with target" (pr-str target))
+
+ ;; Create schema if needed
+ (when-not (driver/schema-exists? driver db (:schema target))
+ (driver/create-schema-if-needed! driver (:conn-spec transform-details) (:schema target)))
+
+ ;; Check cancellation before running query
+ (when (and cancelled? (cancelled?))
+ (throw (ex-info "Transform cancelled before query execution" {:status :cancelled})))
+
+ ;; Run the actual transform
+ (let [result (driver/run-transform! driver transform-details opts)]
+
+ ;; Check cancellation after query
+ (when (and cancelled? (cancelled?))
+ (throw (ex-info "Transform cancelled after query execution" {:status :cancelled})))
+
+ {:status :succeeded
+ :result result}))
+
+ (catch Exception e
+ (let [data (ex-data e)]
+ (if (= :cancelled (:status data))
+ {:status :cancelled
+ :error e}
+ (do
+ (log/error e "Error executing transform")
+ {:status :failed
+ :error e}))))))
+
+;;; ------------------------------------------------- Interface Implementation -------------------------------------------------
+
+(defmethod transforms-base.i/execute-base! :query
+ [transform opts]
+ (run-query-transform! transform opts))
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
new file mode 100644
index 00000000000..3eef908348b
--- /dev/null
+++ b/src/metabase/transforms_base/util.clj
@@ -0,0 +1,731 @@
+(ns metabase.transforms-base.util
+ "Shared utilities for transform execution.
+
+ This namespace contains functions that are used by both query and python transforms
+ and do NOT require transform_run database access."
+ (:require
+ [buddy.core.codecs :as codecs]
+ [buddy.core.hash :as buddy-hash]
+ [clojure.string :as str]
+ [java-time.api :as t]
+ [metabase.driver :as driver]
+ [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.driver.util :as driver.u]
+ [metabase.events.core :as events]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.schema.common :as lib.schema.common]
+ [metabase.query-processor :as qp]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor.middleware.add-remaps :as remap]
+ [metabase.query-processor.middleware.catch-exceptions :as qp.catch-exceptions]
+ [metabase.query-processor.parameters.dates :as params.dates]
+ [metabase.query-processor.preprocess :as qp.preprocess]
+ [metabase.sync.core :as sync]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.util :as u]
+ [metabase.util.date-2 :as u.date]
+ [metabase.util.honey-sql-2 :as h2x]
+ [metabase.util.i18n :as i18n]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
+ [toucan2.core :as t2])
+ (:import
+ (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)
+ (java.util Date)))
+
+(set! *warn-on-reflection* true)
+
+;;; ------------------------------------------------- Constants -------------------------------------------------
+
+(def ^:const transform-temp-table-prefix
+ "Prefix used for temporary tables created during transform execution."
+ "mb_transform_temp_table")
+
+;;; ------------------------------------------------- Transform Type Predicates -------------------------------------------------
+
+(defn transform-type
+ "Get the type of a transform"
+ [transform]
+ (-> transform :source :type keyword))
+
+(defn query-transform?
+ "Check if this is a query transform: native query / mbql query."
+ [transform]
+ (= :query (transform-type transform)))
+
+(defn native-query-transform?
+ "Check if this is a native query transform.
+ Note: The transform should be normalized (via `normalize-transform`) before calling this function."
+ [transform]
+ (when (query-transform? transform)
+ (let [query (-> transform :source :query)]
+ (lib/native-only-query? query))))
+
+(defn python-transform?
+ "Check if this is a Python transform."
+ [transform]
+ (= :python (transform-type transform)))
+
+(defn transform-source-database
+ "Get the source database from a transform"
+ [transform]
+ (case (transform-type transform)
+ :query (-> transform :source :query :database)
+ :python (-> transform :source :source-database)))
+
+;;; ------------------------------------------------- Transform Normalization -------------------------------------------------
+
+(defn normalize-transform
+ "Normalize a transform's source query, similar to how transforms are normalized when read from the database.
+ This should be called on transforms before processing them to ensure queries are in the expected format."
+ [transform]
+ (if (and (map? transform)
+ (= (:type transform) "transform")
+ (get-in transform [:source :query]))
+ (update-in transform [:source :query] lib-be/normalize-query)
+ transform))
+
+(defn transform-source-type
+ "Returns the type of a transform's source: :python, :native, or :mbql.
+ Throws if the source type cannot be detected.
+ Note: The transform should be normalized (via `normalize-transform`) before calling this function."
+ [source]
+ (case (keyword (:type source))
+ :python :python
+ :query (if (lib/native-only-query? (:query source))
+ :native
+ :mbql)
+ (throw (ex-info (str "Unknown transform source type: " (:type source))
+ {:source source}))))
+
+;;; ------------------------------------------------- Feature Checks -------------------------------------------------
+
+(defn required-database-features
+ "Returns the database features necessary to execute `transform`."
+ [transform]
+ (if (python-transform? transform)
+ [:transforms/python]
+ [:transforms/table]))
+
+;;; ------------------------------------------------- Table Names -------------------------------------------------
+
+(defn qualified-table-name
+ "Return the name of the target table of a transform as a possibly qualified symbol."
+ [_driver {:keys [schema name]}]
+ (if schema
+ (keyword schema name)
+ (keyword name)))
+
+(defn temp-table-name
+ "Generate a temporary table name with current timestamp in milliseconds.
+ If table name would exceed max table name length for the driver, fallback to using a shorter timestamp"
+ [driver schema]
+ (let [max-len (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))
+ timestamp (str (System/currentTimeMillis))
+ prefix (str transform-temp-table-prefix "_")
+ available (- max-len (count prefix))
+ ;; If we don't have enough space, take the later digits of the timestamp
+ suffix (if (>= available (count timestamp))
+ timestamp
+ (subs timestamp (- (count timestamp) available)))
+ table-name (str prefix suffix)]
+ (keyword schema table-name)))
+
+;;; ------------------------------------------------- Query Helpers -------------------------------------------------
+
+(defn massage-sql-query
+ "Adjusts mbql query for use in a transform."
+ [query]
+ (-> query
+ remap/disable-remaps
+ lib/disable-default-limit))
+
+;;; ------------------------------------------------- Incremental/Checkpoint Helpers -------------------------------------------------
+
+(defn- checkpoint-incremental?
+ "Returns true if `source` uses checkpoint-based incremental strategy."
+ [source]
+ (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))
+
+(defn supported-incremental-filter-type?
+ "Returns true if the given base-type is supported for incremental filtering.
+
+ We only support temporal (timestamp/tz) and numeric (int/float) types."
+ [base-type]
+ (or (isa? base-type :type/Temporal)
+ (isa? base-type :type/Number)))
+
+(defn- source->checkpoint-filter-unique-key
+ "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."
+ [query source-incremental-strategy]
+ (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))
+
+(defn- source->checkpoint-filter-column
+ "Resolve the checkpoint filter column for an incremental transform.
+
+ Tries to resolve the column using the unique key first.
+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.
+
+ Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).
+ Throws an exception if the column type is not supported."
+ [query source-incremental-strategy table metadata-provider]
+ (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]
+ (when-some [{column-name :name
+ :keys [base-type]
+ :as column}
+ (cond
+ checkpoint-filter-unique-key
+ (source->checkpoint-filter-unique-key query source-incremental-strategy)
+ checkpoint-filter
+ (when-some [field-id (t2/select-one-pk :model/Field
+ :table_id (:id table)
+ :name checkpoint-filter)]
+ (lib.metadata/field metadata-provider field-id)))]
+ (when-not (supported-incremental-filter-type? base-type)
+ (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "
+ "Only numeric and temporal columns are supported for incremental filtering.")
+ {:column-name column-name
+ :base-type base-type})))
+ column)))
+
+(declare target-table)
+
+(defn next-checkpoint
+ "Build a query to compute the MAX of the checkpoint column from the target table.
+
+ Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),
+ or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."
+ [{:keys [source target] :as transform}]
+ (let [db-id (transforms-base.i/target-db-id transform)]
+ (when (checkpoint-incremental? source)
+ (when-let [table (target-table db-id target)]
+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)
+ table-metadata (lib.metadata/table metadata-provider (:id table))
+ query (lib/query metadata-provider table-metadata)]
+ (when-let [filter-column (source->checkpoint-filter-column query
+ (:source-incremental-strategy source)
+ table metadata-provider)]
+ {:query (-> query (lib/aggregate (lib/max filter-column)))
+ :filter-column filter-column}))))))
+
+(defn- next-checkpoint-value
+ "Execute the checkpoint query and normalize the result for database insertion.
+ Returns `nil` if the target table is empty."
+ [{:keys [query filter-column]}]
+ (let [{:keys [base-type]} filter-column
+ v (some-> query qp/process-query :data :rows first first)]
+ ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right
+ ;; shape for reinsertion
+ (cond
+ (nil? v)
+ nil
+
+ (isa? base-type :type/Integer)
+ (bigint v)
+
+ ;; any other number that's not an integer, should be a decimal/float
+ (number? v)
+ (bigdec v)
+
+ :else v)))
+
+(defn preprocess-incremental-query
+ "Add checkpoint checkpoint filtering to a query for incremental execution.
+
+ For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.
+ For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.
+ Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."
+ [query source-incremental-strategy checkpoint]
+ (if-let [checkpoint-value (next-checkpoint-value checkpoint)]
+ (if (lib/native? query)
+ ;; native query with explicit checkpoint filter
+ (if (get-in query [:stages 0 :template-tags "checkpoint"])
+ (update query :parameters conj
+ {:type (if (number? checkpoint-value) :number :text)
+ :target [:variable [:template-tag "checkpoint"]]
+ :value checkpoint-value})
+ query)
+ ;; mbql query
+ (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))
+ query))
+
+(defn- post-process-incremental-query
+ "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.
+
+ Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "
+ [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]
+ (let [{:keys [checkpoint-filter]} source-incremental-strategy]
+ (if (and (lib/native? (:query source))
+ (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))
+ (next-checkpoint-value checkpoint))
+ (let [wrap-query (fn [query]
+ (let [honeysql-query {:select [:*]
+ :from [[[:raw (str "(" query ")")] :subquery]]
+ :where [:> (h2x/identifier :field checkpoint-filter)
+ [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]
+ (first (sql.qp/format-honeysql driver honeysql-query))))]
+ (update outer-query :query wrap-query))
+ outer-query)))
+
+(mu/defn validate-transform-query :- [:maybe [:map [:error :string]]]
+ "Verifies that a query transform's query can actually be run as is. Returns nil on success and an error map on failure."
+ [{:keys [source]}]
+ (case (keyword (:type source))
+ :query
+ (try
+ (qp.preprocess/preprocess (:query source))
+ nil
+ (catch Exception e
+ (qp.catch-exceptions/exception-response e)))))
+
+(defn compile-source
+ "Compile the source query of a transform to SQL, applying incremental filtering if required."
+ [{:keys [source] :as transform}]
+ (let [{:keys [source-incremental-strategy] query-type :type} source]
+ (case (keyword query-type)
+ :query
+ (let [checkpoint (next-checkpoint transform)
+ query (:query source)
+ driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]
+ (binding [driver/*compile-with-inline-parameters*
+ (or (= :clickhouse driver)
+ driver/*compile-with-inline-parameters*)]
+ (-> query
+ (preprocess-incremental-query source-incremental-strategy checkpoint)
+ massage-sql-query
+ qp.compile/compile
+ (post-process-incremental-query driver source checkpoint)))))))
+
+;;; ------------------------------------------------- Target Table Management -------------------------------------------------
+
+(defn target-table
+ "Load the `target` table of a transform from the database specified by `database-id`."
+ [database-id target & kv-args]
+ (some-> (apply t2/select-one :model/Table
+ :db_id database-id
+ :schema (:schema target)
+ :name (:name target)
+ kv-args)
+ (t2/hydrate :db)))
+
+(defn target-table-exists?
+ "Test if the target table of a transform already exists."
+ [{:keys [target] :as transform}]
+ (let [db-id (transforms-base.i/target-db-id transform)
+ {driver :engine :as database} (t2/select-one :model/Database db-id)]
+ (driver/table-exists? driver database target)))
+
+(defn- sync-table!
+ ([database target] (sync-table! database target nil))
+ ([database target {:keys [create?]}]
+ (when-let [table (or (target-table (:id database) target)
+ (when create?
+ (sync/create-table! database (select-keys target [:schema :name :data_source :data_authority]))))]
+ (sync/sync-table! table)
+ table)))
+
+(defn activate-table-and-mark-computed!
+ "Activate table for `target` in `database` in the app db."
+ [database target]
+ (when-let [table (sync-table! database (assoc target
+ :data_authority :computed
+ :data_source :metabase-transform)
+ {:create? true})]
+ (when-not (:active table)
+ (t2/update! :model/Table (:id table) {:active true}))
+ table))
+
+(defn sync-target!
+ "Sync target of a transform"
+ [target database]
+ ;; sync the new table (note that even a failed sync status means that the execution succeeded)
+ (log/info "Syncing target" (pr-str target) "for transform")
+ (activate-table-and-mark-computed! database target))
+
+(defn deactivate-table!
+ "Deactivate table for `target` in `database` in the app db."
+ [database target]
+ (when-let [table (sync-table! database target)]
+ ;; TODO this should probably be a function in the sync module
+ (t2/update! :model/Table (:id table) {:active false})))
+
+(defn delete-target-table!
+ "Delete the target table of a transform and sync it from the app db."
+ [{:keys [id target], :as transform}]
+ (when target
+ (let [target (update target :type keyword)
+ database-id (transforms-base.i/target-db-id transform)]
+ (when database-id
+ (if-let [{driver :engine :as database} (t2/select-one :model/Database database-id)]
+ (do
+ (driver/drop-transform-target! driver database target)
+ (log/info "Deactivating target " (pr-str target) "for transform" id)
+ (deactivate-table! database target))
+ (log/warnf "Skipping drop of transform target %s for transform %d: database %d not found"
+ (pr-str target) id database-id))))))
+
+(defn delete-target-table-by-id!
+ "Delete the target table of the transform specified by `transform-id`."
+ [transform-id]
+ (delete-target-table! (t2/select-one :model/Transform transform-id)))
+
+;;; ------------------------------------------------- Table DDL -------------------------------------------------
+
+(mr/def ::column-definition
+ [:map
+ [:name :string]
+ [:type ::lib.schema.common/base-type]
+ [:nullable? {:optional true} :boolean]])
+
+(mr/def ::table-definition
+ [:map
+ [:name :keyword]
+ [:columns [:sequential ::column-definition]]
+ [:primary-key {:optional true} [:sequential :string]]])
+
+(mu/defn create-table-from-schema!
+ "Create a table from a table-schema"
+ [driver :- :keyword
+ database-id :- pos-int?
+ table-schema :- ::table-definition]
+ (let [{:keys [columns] table-name :name} table-schema
+ column-definitions (mapv (fn [{:keys [name type database-type]}]
+ (let [db-type (if database-type
+ [[:raw database-type]]
+ (try
+ (driver/type->database-type driver type)
+ (catch IllegalArgumentException _
+ (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)
+ (driver/type->database-type driver :type/Text))))]
+ [name db-type]))
+ columns)
+ primary-key-opts (select-keys table-schema [:primary-key])]
+ (log/infof "Creating table %s with %d columns" table-name (count columns))
+ (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))
+
+(defn drop-table!
+ "Drop a table in the database."
+ [driver database-id table-name]
+ (log/infof "Dropping table %s" table-name)
+ (driver/drop-table! driver database-id table-name))
+
+(defn rename-tables!
+ "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.
+ This is a simpler, composable operation that only handles renaming."
+ [driver database-id rename-map]
+ (log/infof "Renaming tables: %s" (pr-str rename-map))
+ (driver/rename-tables! driver database-id rename-map))
+
+;;; ------------------------------------------------- Secondary Index Management -------------------------------------------------
+
+(def ^:private metabase-index-prefix "mb_transform_idx_")
+
+(defn- incremental-filter-index-name [schema table-name filter-column-name]
+ (let [prefix metabase-index-prefix
+ suffix (codecs/bytes->hex (buddy-hash/md5 (str/join "|" [schema table-name filter-column-name])))]
+ (str prefix suffix)))
+
+(defn- should-index-incremental-filter-column? [driver database]
+ (and (driver.u/supports? driver :describe-indexes database)
+ (driver.u/supports? driver :transforms/index-ddl database)))
+
+(defn- metabase-owned-index? [index]
+ (str/starts-with? (:index-name index) metabase-index-prefix))
+
+(defn- metabase-incremental-filter-index [filter-column target]
+ (let [table-name (:name target)
+ schema (:schema target)
+ filter-name (:name filter-column)]
+ ;; match the schema used by describe-table-indexes (:value denotes the column name, assumed leading column if a composite index)
+ {:index-name (incremental-filter-index-name schema table-name filter-name)
+ :value filter-name}))
+
+(defn- decide-secondary-index-ddl
+ "Decides which indexes should be dropped/created on the target table.
+
+ e.g. Indexing the incremental :filter-column to accelerate MAX(target.checkpoint) queries on OLTP databases.
+
+ Indexes are represented as maps with at least the keys :index-name, :value.
+ This matches the form used by [[metabase.driver/describe-table-indexes]].
+
+ Returns a map:
+ :drop - a vector of indexes that are redundant and should be dropped.
+ :create - a vector of desirable indexes that do not exist and that should be created.
+
+ Notes:
+ - If user covering indexes exist they should be reused.
+ - Will never drop user indexes.
+ - Indexes we previously created and are no longer required are dropped."
+ [{:keys [filter-column indexes target database]}]
+ (let [[mb-indexes user-indexes] ((juxt filter remove) metabase-owned-index? indexes)
+ default-mb-index (when filter-column (metabase-incremental-filter-index filter-column target))
+ column-name (:name filter-column)
+ existing-user-index (first (filter #(= (:value %) column-name) user-indexes))
+ existing-mb-index (first (filter #(= (:index-name %) (:index-name default-mb-index)) mb-indexes))
+ driver (:engine database)
+ intended-index (when (should-index-incremental-filter-column? driver database)
+ ;; Prefer reuse to not create redundant indexes
+ (or existing-user-index default-mb-index))
+ drop (if intended-index
+ (remove #(= (:index-name intended-index) (:index-name %)) mb-indexes)
+ mb-indexes)
+ create (when (and intended-index
+ (not= (:index-name intended-index) (:index-name existing-user-index))
+ (not= (:index-name intended-index) (:index-name existing-mb-index)))
+ [intended-index])]
+ {:drop (vec drop)
+ :create (vec create)}))
+
+(defn execute-secondary-index-ddl-if-required!
+ "If target table index modifications are required, executes those CREATE/DROP commands.
+ See [[decide-secondary-index-ddl]] for details.
+
+ `run-id` is optional - if nil, instrumentation/timing is skipped.
+ `with-stage-timing-fn` is optional - if provided, will be called as (with-stage-timing-fn run-id stage thunk)."
+ ([transform database target]
+ (execute-secondary-index-ddl-if-required! transform nil database target nil))
+ ([transform run-id database target with-stage-timing-fn]
+ (when (driver.u/supports? (:engine database) :describe-indexes database)
+ (let [driver (:engine database)
+ indexes (driver/describe-table-indexes driver database target)
+ checkpoint (next-checkpoint transform)
+ {:keys [drop create]}
+ (decide-secondary-index-ddl
+ {:filter-column (:filter-column checkpoint)
+ :database database
+ :target target
+ :indexes indexes})
+ maybe-timed (fn [stage thunk]
+ (if (and run-id with-stage-timing-fn)
+ (with-stage-timing-fn run-id stage thunk)
+ (thunk)))]
+ (doseq [{:keys [index-name value]} drop]
+ (maybe-timed [:import :drop-incremental-filter-index]
+ (fn []
+ (log/infof "Dropping secondary index %s(%s) for target %s" index-name value (pr-str target))
+ (driver/drop-index! driver (:id database) (:schema target) (:name target) index-name))))
+ (doseq [{:keys [index-name value]} create]
+ (maybe-timed [:import :create-incremental-filter-index]
+ (fn []
+ (log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))
+ (driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))))
+
+;;; ------------------------------------------------- Post-Execution Completion -------------------------------------------------
+
+(defn complete-execution!
+ "Post-processing steps after a transform has been executed successfully.
+
+ Performs:
+ - Sync target table to AppDB
+ - Set `transform_id` on the target table
+ - Publish Metabase events (unless `:publish-events?` is false)
+ - Create/drop secondary indexes
+
+ This is called after the core execution completes. Callers that use
+ `run-cancelable-transform!` should call this AFTER `succeed-started-run!`
+ to preserve the correct order of operations."
+ [transform opts]
+ (let [{:keys [target]} transform
+ {:keys [run-id with-stage-timing-fn publish-events?]
+ :or {publish-events? true}} opts
+ db-id (transforms-base.i/target-db-id transform)
+ database (t2/select-one :model/Database db-id)]
+ ;; Sync target table
+ (sync-target! target database)
+ ;; Mark the table as owned by this transform
+ (when-let [table (t2/select-one :model/Table
+ :db_id db-id
+ :schema (:schema target)
+ :name (:name target))]
+ (t2/update! :model/Table (:id table) {:transform_id (:id transform)}))
+ ;; Publish event after sync so the table exists in AppDB.
+ (when publish-events?
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id db-id
+ :transform-id (:id transform)
+ :transform-type (keyword (:type target))
+ :output-schema (:schema target)
+ :output-table (qualified-table-name (:engine database) target)}}))
+ ;; Create secondary indexes if needed
+ (execute-secondary-index-ddl-if-required!
+ transform run-id database target with-stage-timing-fn)))
+
+;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------
+
+(def ^:private ^:const batch-lookup-chunk-size
+ "Maximum number of table refs to query at once to avoid SQL query size limits."
+ 100)
+
+(defn batch-lookup-table-ids
+ "Batch lookup table IDs from ref maps. Returns {[db_id schema name] -> table_id}.
+ Queries the exact conjunction of each [database_id, schema, table] triple rather than
+ a Cartesian product of all values. Uses chunking to avoid query size limits."
+ [refs]
+ (when (seq refs)
+ (let [unique-refs (distinct (map (juxt :database_id :schema :table) refs))
+ ref->clause (fn [[db-id schema table-name]]
+ [:and
+ [:= :db_id db-id]
+ (if (some? schema)
+ [:= :schema schema]
+ [:is :schema nil])
+ [:= :name table-name]])
+ fetch-batch (fn [batch]
+ (t2/select-fn->fn (juxt :db_id :schema :name) :id
+ [:model/Table :id :db_id :schema :name]
+ {:where (into [:or] (map ref->clause batch))}))]
+ (into {} (mapcat fetch-batch) (partition-all batch-lookup-chunk-size unique-refs)))))
+
+(defn- source-table-ref->key
+ "Convert a source table ref map to a lookup key [db_id schema name]."
+ [{:keys [database_id schema table]}]
+ [database_id schema table])
+
+(defn- missing-table-id?
+ "Returns true if `v` is a source table ref map that needs table_id lookup."
+ [v]
+ (and (map? v) (nil? (:table_id v))))
+
+(defn normalize-source-tables
+ "Normalize source-tables to consistent map format {:database_id :schema :table :table_id}.
+
+ The old format stored just integer table IDs. New transforms store maps on write.
+ Old data is converted on read via transform-source-out for backwards compatibility.
+
+ Throws if an integer table ID references a non-existent table.
+ Map refs with non-existent tables get nil table_id (resolved later at execute time)."
+ [source-tables]
+ (let [int-table-ids (into #{} (filter int?) (vals source-tables))
+ int-id->metadata (when (seq int-table-ids)
+ (t2/select-pk->fn (fn [{:keys [db_id schema name]}]
+ {:database_id db_id :schema schema :table name})
+ [:model/Table :id :db_id :schema :name]
+ :id [:in int-table-ids]))
+ missing-ids (when (seq int-table-ids)
+ (remove int-id->metadata int-table-ids))
+ refs-needing-id (filter missing-table-id? (vals source-tables))
+ ref-lookup (or (batch-lookup-table-ids refs-needing-id) {})]
+ (when (seq missing-ids)
+ (throw (ex-info (str "Tables not found for ids: " (str/join ", " (sort missing-ids)))
+ {:table_ids (vec missing-ids)})))
+ (update-vals source-tables
+ (fn [v]
+ (cond
+ (int? v) (assoc (int-id->metadata v) :table_id v)
+ (:table_id v) v
+ :else (assoc v :table_id (ref-lookup (source-table-ref->key v))))))))
+
+(defn resolve-source-tables
+ "Resolve source-tables to {alias -> table_id}. Throws if any table not found.
+ For execute time - all entries must resolve to valid table IDs.
+ Handles both integer IDs (old format) and map refs (new format)."
+ [source-tables]
+ (let [needs-lookup (filter missing-table-id? (vals source-tables))
+ lookup (or (batch-lookup-table-ids needs-lookup) {})
+ resolved (u/for-map [[alias v] source-tables]
+ [alias (if (int? v)
+ v
+ (or (:table_id v) (lookup (source-table-ref->key v))))])
+ unresolved (for [[alias table-id] resolved
+ :when (nil? table-id)
+ :let [v (get source-tables alias)]]
+ {:alias alias
+ :table (if-let [schema (:schema v)]
+ (str schema "." (:table v))
+ (:table v))
+ :ref v})]
+ (when (seq unresolved)
+ (throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved)))
+ {:unresolved unresolved
+ :transform-message "Input table not found"})))
+ resolved))
+
+;;; ------------------------------------------------- Timestamp Helpers -------------------------------------------------
+
+(defn ->instant
+ "Convert a temporal value `t` to an Instant in the system timezone."
+ ^Instant [t]
+ (when t
+ (condp instance? t
+ Instant t
+ Date (.toInstant ^Date t)
+ OffsetDateTime (.toInstant ^OffsetDateTime t)
+ ZonedDateTime (.toInstant ^ZonedDateTime t)
+ LocalDateTime (recur (.atZone ^LocalDateTime t (t/zone-id)))
+ String (recur (u.date/parse t))
+ LocalTime (recur (.atDate ^LocalTime t (t/local-date)))
+ OffsetTime (recur (.atDate ^OffsetTime t (t/local-date)))
+ LocalDate (recur (.atStartOfDay ^LocalDate t))
+ (throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")
+ {:temporal t})))))
+
+(defn utc-timestamp-string
+ "Convert the timestamp t to a string encoding the it in the system timezone."
+ [t]
+ (-> t ->instant str))
+
+(defn localize-run-timestamps
+ "Convert the timestamps of a `run` to ISO strings in UTC."
+ [run]
+ (-> run
+ (u/update-some :start_time utc-timestamp-string)
+ (u/update-some :end_time utc-timestamp-string)))
+
+;;; ------------------------------------------------- Filter Transforms -------------------------------------------------
+
+(defn- matching-timestamp?
+ [job field-path {:keys [start end]}]
+ (when-let [field-instant (->instant (get-in job field-path))]
+ (let [parse #(-> % u.date/parse ->instant)]
+ ;; logic here is to find when it's not matching and invert this
+ (not (or (and start (.isBefore field-instant (parse start)))
+ (and end (.isAfter field-instant (parse end))))))))
+
+(defn ->date-field-filter-xf
+ "Returns an xform for a date filter."
+ [field-path filter-value]
+ (if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]
+ (filter #(matching-timestamp? % field-path range))
+ identity))
+
+(defn ->status-filter-xf
+ "Returns an xform for a transform run status filter."
+ [field-path statuses]
+ (if-let [statuses (->> statuses (map keyword) set not-empty)]
+ (filter #(statuses (get-in % field-path)))
+ identity))
+
+(defn ->tag-filter-xf
+ "Returns an xform for a transform tag filter."
+ [field-path tag-ids]
+ (if-let [tag-ids (-> tag-ids set not-empty)]
+ (filter #(some tag-ids (get-in % field-path)))
+ identity))
+
+;;; ------------------------------------------------- Misc -------------------------------------------------
+
+(defn is-temp-transform-table?
+ "Return true when `table` matches the transform temporary table naming pattern."
+ [table]
+ (when-let [table-name (:name table)]
+ (str/starts-with? (u/lower-case-en table-name) transform-temp-table-prefix)))
+
+(defn db-routing-enabled?
+ "Returns whether or not the given database is either a router or destination database"
+ [db-or-id]
+ (or (t2/exists? :model/DatabaseRouter :database_id (u/the-id db-or-id))
+ (some->> (:router-database-id db-or-id)
+ (t2/exists? :model/DatabaseRouter :database_id))))
+
+(defn throw-if-db-routing-enabled!
+ "Throws if the database has routing enabled. Call before any driver operations to get a
+ clear error message rather than a confusing driver-level failure."
+ [transform database]
+ (when (db-routing-enabled? database)
+ (throw (ex-info (i18n/tru "Failed to run the transform ({0}) because the database ({1}) has database routing turned on. Running transforms on databases with db routing enabled is not supported."
+ (:name transform)
+ (:name database))
+ {:driver (:engine database), :database database}))))
diff --git a/src/metabase/transforms_inspector/context.clj b/src/metabase/transforms_inspector/context.clj
index 28672555bf7..27d6df18c56 100644
--- a/src/metabase/transforms_inspector/context.clj
+++ b/src/metabase/transforms_inspector/context.clj
@@ -6,10 +6,10 @@
[metabase.driver :as driver]
[metabase.lib.core :as lib]
[metabase.query-processor.preprocess :as qp.preprocess]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms-inspector.query-analysis :as query-analysis]
[metabase.transforms-inspector.schema :as transforms-inspector.schema]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.util :as transforms.util]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -30,13 +30,13 @@
"Extract source table information for a transform.
Returns a seq of maps with :table-id, :table-name, :schema, and :db-id."
{:arglists '([transform])}
- (fn [transform] (transforms.util/transform-source-type (:source transform))))
+ (fn [transform] (transforms-base.u/transform-source-type (:source transform))))
(defmethod extract-sources :mbql
[{:keys [source]}]
(try
(let [query (-> (:query source)
- transforms.util/massage-sql-query
+ transforms-base.u/massage-sql-query
qp.preprocess/preprocess)
table-ids (lib/all-source-table-ids query)]
(table-ids->source-info table-ids))
@@ -48,9 +48,9 @@
[{:keys [source] :as transform}]
(try
(let [query (-> (:query source)
- transforms.util/massage-sql-query
+ transforms-base.u/massage-sql-query
qp.preprocess/preprocess)
- db-id (transforms.util/transform-source-database transform)
+ db-id (transforms-base.u/transform-source-database transform)
driver (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)
deps (driver/native-query-deps driver query)
table-ids (keep :table deps)]
@@ -63,7 +63,7 @@
[transform]
(try
(let [source-tables (get-in transform [:source :source-tables])
- normalized (transforms.util/normalize-source-tables source-tables)
+ normalized (transforms-base.u/normalize-source-tables source-tables)
table-ids (keep (fn [[_ v]] (:table_id v)) normalized)]
(table-ids->source-info table-ids))
(catch Exception e
@@ -79,8 +79,8 @@
(defn- get-target-table
"Get the target table for a transform. Returns nil if the target doesn't exist."
[{:keys [target] :as transform}]
- (let [db-id (transforms.i/target-db-id transform)]
- (transforms.util/target-table db-id target)))
+ (let [db-id (transforms-base.i/target-db-id transform)]
+ (transforms-base.u/target-table db-id target)))
;;; -------------------------------------------------- Field Metadata --------------------------------------------------
@@ -250,7 +250,7 @@
(mu/defn build-context :- ::context
"Build context for lens discovery and generation."
[transform]
- (let [source-type (transforms.util/transform-source-type (:source transform))
+ (let [source-type (transforms-base.u/transform-source-type (:source transform))
sources-info (mapv build-table-info (extract-sources transform))
target-table (get-target-table transform)
target-info (when target-table
@@ -259,7 +259,7 @@
join-structure (:join-structure query-info)
column-matches (when (and (seq sources-info) target-info)
(seq (match-columns sources-info target-info query-info)))
- db-id (transforms.util/transform-source-database transform)]
+ db-id (transforms-base.u/transform-source-database transform)]
(cond-> {:source-type source-type
:sources sources-info
:target target-info
diff --git a/src/metabase/transforms_inspector/query_analysis.clj b/src/metabase/transforms_inspector/query_analysis.clj
index 7b51591e138..746e3f22b6d 100644
--- a/src/metabase/transforms_inspector/query_analysis.clj
+++ b/src/metabase/transforms_inspector/query_analysis.clj
@@ -17,7 +17,7 @@
[metabase.lib.core :as lib]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.sql-tools.core :as sql-tools]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.log :as log]
@@ -81,7 +81,7 @@
Returns {:join-structure [...] :visited-fields {...}} or nil on failure."
[transform]
(let [preprocessed (-> transform :source :query
- transforms.util/massage-sql-query
+ transforms-base.u/massage-sql-query
qp.preprocess/preprocess)]
(when (<= (count (:stages preprocessed)) 1)
{:preprocessed-query preprocessed
@@ -232,10 +232,10 @@
[transform sources]
(try
(let [native-query (-> (get-in transform [:source :query])
- transforms.util/massage-sql-query
+ transforms-base.u/massage-sql-query
qp.preprocess/preprocess)
sql (lib/raw-native-query native-query)
- db-id (transforms.util/transform-source-database transform)
+ db-id (transforms-base.u/transform-source-database transform)
driver (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)
parsed (macaw-parsed-query sql driver)
ast (macaw.ast/->ast parsed {:with-instance? false})]
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 9a2ed58498c..29a1523e572 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -13,13 +13,14 @@
[metabase.request.core :as request]
[metabase.server.core :as server]
[metabase.sql-tools.core :as sql-tools]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms-inspector.core :as inspector]
[metabase.transforms-inspector.schema :as inspector.schema]
[metabase.transforms-rest.api.transform-job]
[metabase.transforms-rest.api.transform-tag]
[metabase.transforms.core :as transforms.core]
[metabase.transforms.schema :as transforms.schema]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms.util :as transforms.u]
[metabase.util.i18n :refer [deferred-tru]]
[metabase.util.jvm :as u.jvm]
[metabase.util.malli.registry :as mr]
@@ -169,12 +170,12 @@
(transforms.core/check-feature-enabled! body)
(transforms.core/validate-incremental-column-type! body)
- (api/check (not (transforms.util/target-table-exists? body))
+ (api/check (not (transforms-base.u/target-table-exists? body))
403
(deferred-tru "A table with that name already exists."))
(-> (transforms.core/create-transform! body)
transforms.core/python-source-table-ref->table-id
- transforms.util/add-source-readable))
+ transforms.u/add-source-readable))
(api.macros/defendpoint :get "/:id" :- TransformResponse
"Get a specific transform."
@@ -193,7 +194,7 @@
dependencies (map id->transform dep-ids)]
(->> (t2/hydrate dependencies :creator :owner)
(mapv transforms.core/python-source-table-ref->table-id)
- transforms.util/add-source-readable)))
+ transforms.u/add-source-readable)))
(def ^:private MergeHistoryEntry
[:map
@@ -244,7 +245,7 @@
(-> (transforms.core/paged-runs (assoc query-params
:offset (request/offset)
:limit (request/limit)))
- (update :data #(map transforms.util/localize-run-timestamps %))))
+ (update :data #(map transforms-base.u/localize-run-timestamps %))))
(api.macros/defendpoint :get "/run/:run-id" :- TransformRunResponse
"Get a transform run by ID."
@@ -253,7 +254,7 @@
(api/check-data-analyst)
(let [run (api/check-404 (t2/select-one :model/TransformRun :id run-id))]
(-> (t2/hydrate run [:transform :collection :transform_tag_ids])
- transforms.util/localize-run-timestamps)))
+ transforms-base.u/localize-run-timestamps)))
(api.macros/defendpoint :put "/:id" :- TransformResponse
"Update a transform."
@@ -284,7 +285,7 @@
[{:keys [id]} :- [:map
[:id ms/PositiveInt]]]
(api/write-check :model/Transform id)
- (transforms.util/delete-target-table-by-id! id)
+ (transforms-base.u/delete-target-table-by-id! id)
nil)
(api.macros/defendpoint :post "/:id/cancel" :- :nil
@@ -294,7 +295,7 @@
(let [transform (api/write-check :model/Transform id)
run (api/check-404 (transforms.core/running-run-for-transform-id id))]
(transforms.core/mark-cancel-started-run! (:id run))
- (when (transforms.util/python-transform? transform)
+ (when (transforms-base.u/python-transform? transform)
(transforms.core/cancel-run! (:id run))))
nil)
diff --git a/src/metabase/transforms_rest/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.clj
index 01758de2184..92ca02ad8e4 100644
--- a/src/metabase/transforms_rest/api/transform_job.clj
+++ b/src/metabase/transforms_rest/api/transform_job.clj
@@ -5,8 +5,9 @@
[metabase.api.macros :as api.macros]
[metabase.api.routes.common :refer [+auth]]
[metabase.models.interface :as mi]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.core :as transforms.core]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms.util :as transforms.u]
[metabase.util.i18n :refer [deferred-tru LocalizedString]]
[metabase.util.jvm :as u.jvm]
[metabase.util.log :as log]
@@ -203,7 +204,7 @@
(defn- add-next-run
[{id :id :as job}]
(if-let [start-time (-> id transforms.core/existing-trigger :next-fire-time)]
- (assoc job :next_run {:start_time (str (transforms.util/->instant start-time))})
+ (assoc job :next_run {:start_time (str (transforms-base.u/->instant start-time))})
job))
(api.macros/defendpoint :get "/:job-id/transforms" :- [:sequential TransformResponse]
@@ -215,7 +216,7 @@
(-> (transforms.core/job-transforms job-id)
(#(do (api/check-403 (every? mi/can-read? %)) %))
(t2/hydrate :creator)
- transforms.util/add-source-readable))
+ transforms.u/add-source-readable))
(api.macros/defendpoint :get "/" :- [:sequential TransformJobResponse]
"Get all transform jobs."
@@ -231,12 +232,12 @@
(let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})]
(into []
(comp (map add-next-run)
- (transforms.util/->date-field-filter-xf [:last_run :start_time] last-run-start-time)
- (transforms.util/->date-field-filter-xf [:next_run :start_time] next-run-start-time)
- (transforms.util/->status-filter-xf [:last_run :status] last-run-statuses)
- (transforms.util/->tag-filter-xf [:tag_ids] tag-ids)
- (map #(update % :last_run transforms.util/localize-run-timestamps))
- (map #(update % :next_run transforms.util/localize-run-timestamps)))
+ (transforms-base.u/->date-field-filter-xf [:last_run :start_time] last-run-start-time)
+ (transforms-base.u/->date-field-filter-xf [:next_run :start_time] next-run-start-time)
+ (transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)
+ (transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)
+ (map #(update % :last_run transforms-base.u/localize-run-timestamps))
+ (map #(update % :next_run transforms-base.u/localize-run-timestamps)))
(t2/hydrate jobs :tag_ids :last_run))))
(def ^{:arglists '([request respond raise])} routes
diff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.clj
index dd86df90910..7026ab93ed0 100644
--- a/test/metabase/transforms/execute_test.clj
+++ b/test/metabase/transforms/execute_test.clj
@@ -11,11 +11,11 @@
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
[metabase.test.data.sql :as sql.tx]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.execute :as transforms.execute]
[metabase.transforms.query-test-util :as query-test-util]
[metabase.transforms.test-dataset :as transforms-dataset]
[metabase.transforms.test-util :as transforms.tu :refer [delete-schema! with-transform-cleanup!]]
- [metabase.transforms.util :as transforms.util]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -140,7 +140,7 @@
{:name "name" :type :type/Text :nullable? true}]}]
(mt/as-admin
(testing "create-table-from-schema! should create the table successfully"
- (transforms.util/create-table-from-schema! driver db-id table-schema)
+ (transforms-base.u/create-table-from-schema! driver db-id table-schema)
(let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})]
(is (some? table-exists?) "Table should exist in the database schema")
(driver/drop-table! driver db-id (:name table-schema))))))))
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index c96ae368705..a13f664e4c6 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -152,12 +152,12 @@
run-id 102
logged-messages (atom [])
run-called? (atom false)]
- (with-redefs [log/log* (fn [_ level _ message]
- (swap! logged-messages conj {:level level :message message}))
- transform-run/running-run-for-transform-id (constantly nil)
- transforms.execute/execute! (fn [_ _]
- (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]
+ (swap! logged-messages conj {:level level :message message}))
+ transform-run/running-run-for-transform-id (constantly nil)
+ transforms.execute/execute! (fn [_ _]
+ (reset! run-called? true))
+ transforms.job-run/add-run-activity! (constantly nil)]
(#'jobs/run-transform! run-id :scheduled nil query-transform)
(is (empty? (filter (comp #{:warn} :level) @logged-messages))
"Should not log warnings when feature is enabled")
@@ -170,9 +170,9 @@
:name "Test Query Transform"}
run-id 100
logged-messages (atom [])]
- (with-redefs [log/log* (fn [_ level _ message]
- (swap! logged-messages conj {:level level :message message}))
- transform-run/running-run-for-transform-id (constantly nil)]
+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]
+ (swap! logged-messages conj {:level level :message message}))
+ transform-run/running-run-for-transform-id (constantly nil)]
(#'jobs/run-transform! run-id :scheduled nil query-transform)
(is (= 1 (count @logged-messages))
"Should log exactly one warning")
@@ -190,12 +190,12 @@
run-id 102
logged-messages (atom [])
run-called? (atom false)]
- (with-redefs [log/log* (fn [_ level _ message]
- (swap! logged-messages conj {:level level :message message}))
- transform-run/running-run-for-transform-id (constantly nil)
- transforms.execute/execute! (fn [_ _]
- (reset! run-called? true))
- transforms.job-run/add-run-activity! (constantly nil)]
+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]
+ (swap! logged-messages conj {:level level :message message}))
+ transform-run/running-run-for-transform-id (constantly nil)
+ transforms.execute/execute! (fn [_ _]
+ (reset! run-called? true))
+ transforms.job-run/add-run-activity! (constantly nil)]
(#'jobs/run-transform! run-id :scheduled nil query-transform)
(is (empty? (filter (comp #{:warn} :level) @logged-messages))
"Should not log warnings when feature is enabled")
@@ -237,9 +237,9 @@
:tag_id (:id tag)
:position 0}]
(let [run-id-atom (atom nil)]
- (with-redefs [jobs/run-transforms! (fn [run-id & _]
- (reset! run-id-atom run-id)
- (throw (ex-info "Uncaught error" {})))]
+ (mt/with-dynamic-fn-redefs [jobs/run-transforms! (fn [run-id & _]
+ (reset! run-id-atom run-id)
+ (throw (ex-info "Uncaught error" {})))]
(try
(jobs/run-job! (:id job) {:run-method :cron})
(catch clojure.lang.ExceptionInfo _))
@@ -286,9 +286,9 @@
:tag_id (:id tag)
:position 0}]
(let [run-id-atom (atom nil)]
- (with-redefs [jobs/run-transforms! (fn [run-id & _]
- (reset! run-id-atom run-id)
- (throw (ex-info "Uncaught error" {})))]
+ (mt/with-dynamic-fn-redefs [jobs/run-transforms! (fn [run-id & _]
+ (reset! run-id-atom run-id)
+ (throw (ex-info "Uncaught error" {})))]
(try
(jobs/run-job! (:id job) {:run-method :manual})
(catch clojure.lang.ExceptionInfo _))
@@ -481,14 +481,14 @@
on-enter (fn [] (await-barrier barrier-enter))
on-exit (fn [] (await-barrier barrier-exit) (.set tl-tripped true))
original-insert transforms.u/try-start-unless-already-running]
- (with-redefs [transforms.u/try-start-unless-already-running
- (fn [transform-id run-method user-id]
- (on-enter)
- (let [[ret ex] (try
- [(original-insert transform-id run-method user-id)]
- (catch Throwable t [nil t]))]
- (on-exit)
- (if ex (throw ex) ret)))]
+ (mt/with-dynamic-fn-redefs [transforms.u/try-start-unless-already-running
+ (fn [transform-id run-method user-id]
+ (on-enter)
+ (let [[ret ex] (try
+ [(original-insert transform-id run-method user-id)]
+ (catch Throwable t [nil t]))]
+ (on-exit)
+ (if ex (throw ex) ret)))]
(let [run1 (transforms.job-run/start-run! (:id job) :manual)
run2 (transforms.job-run/start-run! (:id job) :manual)
fut1 (future
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index a86444bb11a..b307976b702 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -1,17 +1,20 @@
(ns ^:mb/driver-tests metabase.transforms.util-test
"Tests for transform utility functions."
(:require
+ [clojure.core.async :as a]
[clojure.test :refer :all]
[metabase.api.common :as api]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
- [metabase.events.core :as events]
[metabase.lib.core :as lib]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions-group :as perms-group]
[metabase.test :as mt]
[metabase.test.data.sql :as sql.tx]
- [metabase.transforms.util :as transforms.util]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.execute :as transforms.execute]
+ [metabase.transforms.util :as transforms.u]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -64,8 +67,8 @@
(testing "tables with schema"
(let [table-with-schema {:name (name (driver.u/temp-table-name :postgres :schema/orders))}
table-without-schema {:name (name (driver.u/temp-table-name :postgres :orders))}]
- (is (true? (transforms.util/is-temp-transform-table? table-with-schema)))
- (is (true? (transforms.util/is-temp-transform-table? table-without-schema)))))))
+ (is (true? (transforms.u/is-temp-transform-table? table-with-schema)))
+ (is (true? (transforms.u/is-temp-transform-table? table-without-schema)))))))
(deftest create-table-from-schema!-test
(testing "create-table-from-schema! preserves column order from schema definition"
@@ -85,7 +88,7 @@
(mt/as-admin
(try
(testing "Creating table with ordered columns"
- (transforms.util/create-table-from-schema! driver db-id table-schema)
+ (transforms-base.u/create-table-from-schema! driver db-id table-schema)
(is (driver/table-exists? driver (mt/db) {:schema schema-name :name (name table-name)})))
(when (get-method driver/describe-table driver)
@@ -112,7 +115,7 @@
(deftest ^:parallel matching-timestamp?-test
(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"
- (let [matching-timestamp? #'transforms.util/matching-timestamp?
+ (let [matching-timestamp? #'transforms-base.u/matching-timestamp?
field-path [:start_time]
range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}
range-start-only {:start "2024-01-01T00:00:00Z" :end nil}
@@ -155,30 +158,30 @@
:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}
:model/Table t2 {:db_id (:id db) :name "table_two" :schema "my_schema"}]
(testing "returns nil for empty input"
- (is (nil? (transforms.util/batch-lookup-table-ids [])))
- (is (nil? (transforms.util/batch-lookup-table-ids nil))))
+ (is (nil? (transforms-base.u/batch-lookup-table-ids [])))
+ (is (nil? (transforms-base.u/batch-lookup-table-ids nil))))
(testing "looks up table without schema"
(let [refs [{:database_id (:id db) :schema nil :table "table_one"}]
- result (transforms.util/batch-lookup-table-ids refs)]
+ result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {[(:id db) nil "table_one"] (:id t1)} result))))
(testing "looks up table with schema"
(let [refs [{:database_id (:id db) :schema "my_schema" :table "table_two"}]
- result (transforms.util/batch-lookup-table-ids refs)]
+ result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {[(:id db) "my_schema" "table_two"] (:id t2)} result))))
(testing "handles mixed refs with and without schema"
(let [refs [{:database_id (:id db) :schema nil :table "table_one"}
{:database_id (:id db) :schema "my_schema" :table "table_two"}]
- result (transforms.util/batch-lookup-table-ids refs)]
+ result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {[(:id db) nil "table_one"] (:id t1)
[(:id db) "my_schema" "table_two"] (:id t2)}
result))))
(testing "returns empty for non-existent table"
(let [refs [{:database_id (:id db) :schema nil :table "nonexistent"}]
- result (transforms.util/batch-lookup-table-ids refs)]
+ result (transforms-base.u/batch-lookup-table-ids refs)]
(is (= {} result)))))))
(deftest normalize-source-tables-test
@@ -186,7 +189,7 @@
(mt/with-temp [:model/Database db {}
:model/Table t1 {:db_id (:id db) :name "existing_table" :schema nil}]
(testing "converts integer table ID to map format"
- (let [result (transforms.util/normalize-source-tables {"t" (:id t1)})]
+ (let [result (transforms-base.u/normalize-source-tables {"t" (:id t1)})]
(is (map? (get result "t")))
(is (= (:id db) (get-in result ["t" :database_id])))
(is (= "existing_table" (get-in result ["t" :table])))
@@ -194,27 +197,27 @@
(testing "throws for non-existent integer table ID"
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found for ids: 999999"
- (transforms.util/normalize-source-tables {"t" 999999}))))
+ (transforms-base.u/normalize-source-tables {"t" 999999}))))
(testing "populates table_id for existing table"
(let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table"}}
- result (transforms.util/normalize-source-tables source-tables)]
+ result (transforms-base.u/normalize-source-tables source-tables)]
(is (= (:id t1) (get-in result ["t" :table_id])))))
(testing "preserves existing table_id"
(let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table" :table_id 999}}
- result (transforms.util/normalize-source-tables source-tables)]
+ result (transforms-base.u/normalize-source-tables source-tables)]
(is (= 999 (get-in result ["t" :table_id])))))
(testing "leaves table_id nil for non-existent table ref"
(let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}
- result (transforms.util/normalize-source-tables source-tables)]
+ result (transforms-base.u/normalize-source-tables source-tables)]
(is (nil? (get-in result ["t" :table_id])))))
(testing "handles mixed int and map entries"
(let [source-tables {"t1" (:id t1)
"t2" {:database_id (:id db) :schema nil :table "existing_table"}}
- result (transforms.util/normalize-source-tables source-tables)]
+ result (transforms-base.u/normalize-source-tables source-tables)]
(is (map? (get result "t1")))
(is (= (:id t1) (get-in result ["t1" :table_id])))
(is (= (:id t1) (get-in result ["t2" :table_id]))))))))
@@ -225,31 +228,31 @@
:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}
:model/Table t2 {:db_id (:id db) :name "table_two" :schema nil}]
(testing "passes through integer entries (old format)"
- (is (= {"t" 123} (transforms.util/resolve-source-tables {"t" 123}))))
+ (is (= {"t" 123} (transforms-base.u/resolve-source-tables {"t" 123}))))
(testing "resolves map with table_id"
(let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}}]
- (is (= {"t" (:id t1)} (transforms.util/resolve-source-tables source-tables)))))
+ (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))
(testing "looks up table_id for map without it"
(let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one"}}]
- (is (= {"t" (:id t1)} (transforms.util/resolve-source-tables source-tables)))))
+ (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))
(testing "throws for non-existent table"
(let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}]
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: nonexistent"
- (transforms.util/resolve-source-tables source-tables)))))
+ (transforms-base.u/resolve-source-tables source-tables)))))
(testing "throws with schema in error message"
(let [source-tables {"t" {:database_id (:id db) :schema "my_schema" :table "nonexistent"}}]
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: my_schema\.nonexistent"
- (transforms.util/resolve-source-tables source-tables)))))
+ (transforms-base.u/resolve-source-tables source-tables)))))
(testing "handles mixed entries (old and new format)"
(let [source-tables {"t1" (:id t1)
"t2" {:database_id (:id db) :schema nil :table "table_two"}}]
(is (= {"t1" (:id t1) "t2" (:id t2)}
- (transforms.util/resolve-source-tables source-tables))))))))
+ (transforms-base.u/resolve-source-tables source-tables))))))))
(deftest source-tables-readable?-test
(testing "source-tables-readable? function"
@@ -262,17 +265,17 @@
(testing "returns true for query transform when user can read database"
(let [transform {:source {:type :query
:query {:database db-id}}}]
- (is (true? (transforms.util/source-tables-readable? transform)))))
+ (is (true? (transforms.u/source-tables-readable? transform)))))
(testing "returns true for python transform when user can read all source tables"
(let [transform {:source {:type :python
:source-tables {"t1" table-id}}}]
- (is (true? (transforms.util/source-tables-readable? transform)))))
+ (is (true? (transforms.u/source-tables-readable? transform)))))
(testing "handles source tables with table_id in map format"
(let [transform {:source {:type :python
:source-tables {"t1" {:table_id table-id}}}}]
- (is (true? (transforms.util/source-tables-readable? transform))))))))))))
+ (is (true? (transforms.u/source-tables-readable? transform))))))))))))
(deftest source-tables-readable-permissions-test
(testing "source-tables-readable? with various permission levels"
@@ -290,7 +293,7 @@
(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked
(mt/with-db-perm-for-group! group db-id :perms/view-data :blocked
(binding [api/*current-user-id* (:id user)]
- (is (false? (transforms.util/source-tables-readable? transform))
+ (is (false? (transforms.u/source-tables-readable? transform))
"User with blocked database access should not be able to read source database")))))))
(testing "Python transforms - blocked database access"
@@ -301,7 +304,7 @@
(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked
(mt/with-db-perm-for-group! group db-id :perms/view-data :blocked
(binding [api/*current-user-id* (:id user)]
- (is (false? (transforms.util/source-tables-readable? transform))
+ (is (false? (transforms.u/source-tables-readable? transform))
"User with blocked database access should not be able to read source tables")))))))
(testing "Python transforms - granular access but missing some tables"
@@ -317,25 +320,31 @@
;; Block table2 for the test group only
(data-perms/set-table-permission! (:id group) table2-id :perms/view-data :blocked)
(binding [api/*current-user-id* (:id user)]
- (is (false? (transforms.util/source-tables-readable? transform))
+ (is (false? (transforms.u/source-tables-readable? transform))
"User who cannot read all source tables should have source_readable=false")))))))))))))
-(deftest handle-transform-complete-sets-transform-id-test
- (testing "handle-transform-complete! sets transform_id on the target table"
- (mt/with-premium-features #{:transforms}
- (let [target {:type "table" :schema nil :name "test_output_table"}]
- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :h2}
- :model/Transform {transform-id :id :as transform} {:target target}
- :model/Table {table-id :id :as table} {:db_id db-id :name "test_output_table" :schema nil}
- :model/TransformRun {run-id :id} {:transform_id transform-id
- :status "running"
- :run_method "manual"}]
- (with-redefs [transforms.util/sync-target! (constantly table)
- events/publish-event! (constantly nil)
- transforms.util/execute-secondary-index-ddl-if-required! (constantly nil)]
- (transforms.util/handle-transform-complete! :run-id run-id :transform transform :db db)
- (is (= transform-id
- (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))
+(deftest execute-sets-transform-id-on-target-table-test
+ (testing "Executing a query transform sets transform_id on the target table"
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/with-premium-features #{:transforms}
+ (let [target {:type "table" :schema nil :name "test_output_table"}]
+ (mt/with-temp [:model/Transform {transform-id :id :as transform}
+ {:target target
+ :source {:type "query"
+ :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}
+ :model/Table {table-id :id} {:db_id (mt/id) :name "test_output_table" :schema nil}]
+ ;; Mock execute-base! to return success without actually running a query,
+ ;; run-cancelable-transform! to bypass schema creation / cancellation infra,
+ ;; and sync/indexes to skip driver calls. complete-execution! runs normally
+ ;; so it sets transform_id on the target table.
+ (with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})
+ transforms-base.u/sync-target! (constantly nil)
+ transforms-base.u/execute-secondary-index-ddl-if-required! (constantly nil)
+ transforms.u/run-cancelable-transform! (fn [_run-id _driver _details run-fn & _opts]
+ (run-fn (a/promise-chan)))]
+ (transforms.execute/execute! transform {:run-method :manual})
+ (is (= transform-id
+ (t2/select-one-fn :transform_id :model/Table :id table-id))))))))))
(deftest transform-hydration-test
(testing "hydrating :transform on a table"
@@ -356,7 +365,7 @@
(deftest ^:parallel massage-sql-query-test
(testing "massage-sql-query sets disable-remaps? and disable-max-results?"
(let [query {:database 1, :type :query, :query {:source-table 1}}
- massaged (transforms.util/massage-sql-query query)]
+ massaged (transforms-base.u/massage-sql-query query)]
(is (true? (get-in massaged [:middleware :disable-remaps?])))
(is (true? (get-in massaged [:middleware :disable-max-results?]))))))
@@ -366,7 +375,7 @@
(mt/with-premium-features #{:transforms}
(let [transform {:source {:type "query"
:query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}
- {:keys [query]} (transforms.util/compile-source transform)]
+ {:keys [query]} (transforms-base.u/compile-source transform)]
(is (string? query))
(is (not (re-find #"(?i)\bLIMIT\b" query))
(str "Expected no LIMIT clause in compiled SQL, got: " query)))))))
diff --git a/test/metabase/transforms/ordering_test.clj b/test/metabase/transforms_base/ordering_test.clj
similarity index 98%
rename from test/metabase/transforms/ordering_test.clj
rename to test/metabase/transforms_base/ordering_test.clj
index 6cc78a2910f..c390b54bfcf 100644
--- a/test/metabase/transforms/ordering_test.clj
+++ b/test/metabase/transforms_base/ordering_test.clj
@@ -1,4 +1,4 @@
-(ns ^:mb/driver-tests metabase.transforms.ordering-test
+(ns ^:mb/driver-tests metabase.transforms-base.ordering-test
(:require
[clojure.test :refer :all]
[metabase.driver :as driver]
@@ -6,9 +6,9 @@
[metabase.query-processor.compile :as qp.compile]
[metabase.test :as mt]
[metabase.test.data.sql :as sql.tx]
+ [metabase.transforms-base.interface :as transforms-base.i]
+ [metabase.transforms-base.ordering :as ordering]
[metabase.transforms.execute :as transforms.execute]
- [metabase.transforms.interface :as transforms.i]
- [metabase.transforms.ordering :as ordering]
[toucan2.core :as t2]))
(defn- default-schema-or-public [& [fallback-driver]]
@@ -58,7 +58,7 @@
(defn- transform-deps-for-db [transform]
(mt/with-metadata-provider (mt/id)
- (#'transforms.i/table-dependencies transform)))
+ (transforms-base.i/table-dependencies transform)))
(deftest not-run-transform-dependency-ordering-test
(mt/test-driver (mt/normal-driver-select {:+parent :sql-jdbc})
diff --git a/test/metabase/transforms_inspector/lens_output_test.clj b/test/metabase/transforms_inspector/lens_output_test.clj
index 41453dba6d1..0cc4de3818b 100644
--- a/test/metabase/transforms_inspector/lens_output_test.clj
+++ b/test/metabase/transforms_inspector/lens_output_test.clj
@@ -6,9 +6,9 @@
[metabase.lib.metadata :as lib.metadata]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.test :as mt]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms-inspector.core :as inspector.core]
[metabase.transforms-inspector.lens.core :as lens.core]
- [metabase.transforms.util :as transforms.util]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -52,7 +52,7 @@
(lib/with-join-alias "Products")))])
(lib/with-join-alias "Products")
(lib/with-join-fields :all)))
- transforms.util/massage-sql-query
+ transforms-base.u/massage-sql-query
qp.preprocess/preprocess)))
(defn- extract-join-structure
@@ -100,7 +100,7 @@
(let [orders-info (build-table-info (mt/id :orders))
mp (mt/metadata-provider)
preprocessed (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))
- transforms.util/massage-sql-query
+ transforms-base.u/massage-sql-query
qp.preprocess/preprocess)]
{:transform {}
:source-type :mbql
diff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.clj
index 3d35e9aa173..08a232bb38d 100644
--- a/test/metabase/transforms_rest/api/transform_test.clj
+++ b/test/metabase/transforms_rest/api/transform_test.clj
@@ -12,6 +12,7 @@
[metabase.query-processor :as qp]
[metabase.search.test-util :as search.tu]
[metabase.test :as mt]
+ [metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms-rest.api.transform]
[metabase.transforms.crud]
[metabase.transforms.query-test-util :as query-test-util]
@@ -20,7 +21,6 @@
parse-instant
utc-timestamp
with-transform-cleanup!]]
- [metabase.transforms.util :as transforms.util]
[metabase.util :as u]
[toucan2.core :as t2]))
@@ -647,7 +647,7 @@
(m/dissoc-in [:source :query :lib/metadata]))
(-> (mt/user-http-request :lucky :put 200 (format "transform/%s" (:id resp)) updated)
(update-in [:source :query] lib/normalize))))
- (is (false? (transforms.util/target-table-exists? original)))))))))))
+ (is (false? (transforms-base.u/target-table-exists? original)))))))))))
(deftest delete-transforms-test
(mt/with-premium-features #{}
@@ -776,7 +776,7 @@
original)
_ (do (test-run! transform-id)
(wait-for-table table1-name 5000))
- _ (is (true? (transforms.util/target-table-exists? original)))
+ _ (is (true? (transforms-base.u/target-table-exists? original)))
_ (check-query-results table1-name [5 11 16] "Gadget")
updated {:name "Doohickey Products"
:description "Desc"
@@ -790,8 +790,8 @@
(update-in [:source :query] lib/normalize))))
(test-run! transform-id)
(wait-for-table table2-name 5000)
- (is (true? (transforms.util/target-table-exists? original)))
- (is (true? (transforms.util/target-table-exists? updated)))
+ (is (true? (transforms-base.u/target-table-exists? original)))
+ (is (true? (transforms-base.u/target-table-exists? updated)))
(check-query-results table2-name [2 3 4 13] "Doohickey")))))))))))
(deftest get-runs-filter-by-single-transform-id-test