The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Extract non-appdb module out of transforms (attempt #3) (#70245)
Chris Truter57586f242722
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 121ab6d0dec..08727d8db1b 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -422,6 +422,8 @@ring.util.codec codecring.util.response responsesaml20-clj.core saml+ metabase.transforms-base.util transforms-base.u+ metabase.transforms.util transforms.utoucan2.core t2}}}:lint-asdiff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex e320b1048d7..bd37fbb390b 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -870,6 +870,7 @@metabase.models.interfacemetabase.models.resolutionmetabase.models.serialization+ metabase.models.transforms.transformmetabase.models.util.spec-updatemetabase.models.visualization-settings}:uses #{api@@ -886,6 +887,7 @@searchsettingstransforms+ transforms-baseutil}:friends #{transforms}}@@ -1574,7 +1576,6 @@:api #{metabase.transforms.coremetabase.transforms.feature-gatingmetabase.transforms.init- metabase.transforms.interfacemetabase.transforms.schemametabase.transforms.util}:uses #{analytics@@ -1584,21 +1585,36 @@drivereventslib- lib-bemodelspremium-featuresqueriesquery-processorrevisionssettings- synctasktask-history+ transforms-baseutil}:friends #{modelsenterprise/transforms-pythonenterprise/workspaces}}+ transforms-base+ {:team "Querying"+ :api #{metabase.transforms-base.init+ metabase.transforms-base.interface+ metabase.transforms-base.util}+ :uses #{driver+ events+ lib+ lib-be+ query-processor+ sync+ util}+ :friends #{transforms+ enterprise/transforms-python+ enterprise/workspaces}}+transforms-inspector{:team "Gadget":api #{metabase.transforms-inspector.core@@ -1608,7 +1624,7 @@lib-bequery-processorsql-tools- transforms+ transforms-baseutil}}transforms-rest@@ -1625,6 +1641,7 @@serversql-toolstransforms+ transforms-basetransforms-inspectorutil}}@@ -2136,6 +2153,7 @@settingstasktransforms+ transforms-baseutil}}enterprise/email@@ -2253,6 +2271,7 @@systemtasktransforms+ transforms-baseusersutilwarehouses@@ -2476,6 +2495,7 @@query-processorsettingstransforms+ transforms-baseutil}}enterprise/upload-management@@ -2509,6 +2529,7 @@sql-toolssystemtransforms+ transforms-baseutilenterprise/transforms-python}}}diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex e89f8e6d18e..21511052820 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -7,7 +7,7 @@[metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.queries.schema :as queries.schema]- [metabase.transforms.core :as transforms]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]))@@ -60,7 +60,7 @@[:python;; If the upstream table doesn't exist yet, table_id will be nil[:map [:source-tables {:optional true} [:map-of :string [:or :int [:map [:table_id [:maybe :int]]]]]]]]]]]]- (let [source-type (transforms/transform-type transform)]+ (let [source-type (transforms-base.u/transform-type transform)](case source-type:query (upstream-deps:query query):python (upstream-deps:python-transform transform)diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.cljindex d0be3f46e2e..c1518645217 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/context.clj@@ -8,7 +8,7 @@[metabase.config.core :as config][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu]@@ -146,9 +146,9 @@(try(if (and (= (:type item) "transform")(not (:source_type item)))- (let [transform (transforms.util/normalize-transform item)]+ (let [transform (transforms-base.u/normalize-transform item)](assoc transform- :source_type (transforms.util/transform-source-type (:source transform))))+ :source_type (transforms-base.u/transform-source-type (:source transform))))item)(catch Exception e(log/error e "Error annotating transform source type for metabot context")diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljnew file mode 100644index 00000000000..d0eda8d5e5f--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -0,0 +1,376 @@+(ns metabase-enterprise.transforms-python.base+ "Base Python transform execution - core logic without transform_run tracking.++ This namespace handles Python transform execution and returns results in memory+ rather than writing to transform_run rows. Logs are returned in the result map+ instead of being saved to transform_run.message."+ (:require+ [clojure.core.async :as a]+ [clojure.java.io :as io]+ [clojure.string :as str]+ [metabase-enterprise.transforms-python.python-runner :as python-runner]+ [metabase-enterprise.transforms-python.s3 :as s3]+ [metabase-enterprise.transforms-python.settings :as transforms-python.settings]+ [metabase.driver :as driver]+ [metabase.driver.util :as driver.u]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.instrumentation :as transforms.instrumentation]+ [metabase.util :as u]+ [metabase.util.format :as u.format]+ [metabase.util.i18n :as i18n]+ [metabase.util.log :as log]+ [toucan2.core :as t2])+ (:import+ (java.io File)+ (java.nio.file Files)+ (java.nio.file.attribute FileAttribute)))++(set! *warn-on-reflection* true)++;;; ------------------------------------------------- Interface Implementations -------------------------------------------------++(defmethod transforms-base.i/target-db-id :python+ [transform]+ (-> transform :target :database))++(defmethod transforms-base.i/source-db-id :python+ [transform]+ (-> transform :source :source-database))++(defn- source-table-value->dependency+ "Convert a source table value (int or ref map) to a dependency map."+ [v]+ (cond+ ;; Integer table ID - direct table dependency+ (int? v)+ {:table v}++ ;; Map with resolved table_id - direct table dependency+ (:table_id v)+ {:table (:table_id v)}++ ;; Map without table_id - return ref for ordering system to resolve+ :else+ {:table-ref (select-keys v [:database_id :schema :table])}))++(defmethod transforms-base.i/table-dependencies :python+ [transform]+ (into #{}+ (map source-table-value->dependency)+ (vals (get-in transform [:source :source-tables]))))++;;; ------------------------------------------------- Message Log (in-memory only) -------------------------------------------------++(defn empty-message-log+ "Returns a new message log for accumulating execution logs in memory."+ []+ (atom {:pre-python []+ :python nil+ :post-python []}))++(defn- log!+ "Appends a string to the message log."+ [message-log s]+ (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))+ nil)++(defn- replace-python-logs!+ "Called when events output is received to copy those events into the message-log."+ [message-log events]+ (swap! message-log assoc :python events)+ nil)++(defn message-log->string+ "Presents the content of the log as a string."+ [message-log]+ (let [{:keys [pre-python python post-python]} @message-log]+ (str/join "\n" (concat pre-python+ (map :message python)+ post-python))))++;;; ------------------------------------------------- Transfer Strategies -------------------------------------------------++(defn- table-schema [table-name metadata]+ {:name (if (keyword? table-name) table-name (keyword table-name))+ :columns (mapv (fn [{:keys [name base_type]}]+ {:name name+ :type (python-runner/restricted-insert-type base_type)+ :nullable? true})+ (:fields metadata))})++(defn- insert-data!+ "Insert data from source into an existing table."+ [driver db-id table-schema data-source]+ (let [data-source (assoc data-source :table-schema table-schema)]+ (driver/insert-from-source! driver db-id table-schema data-source)))++(defn- create-table-and-insert-data!+ "Create a table from metadata and insert data from source."+ [driver db-id table-schema data-source]+ (transforms-base.u/create-table-from-schema! driver db-id table-schema)+ (insert-data! driver db-id table-schema data-source))++(defn- transfer-with-rename-tables-strategy!+ "Transfer data using the rename-tables*! multimethod with atomicity guarantees."+ [driver db-id table-name metadata data-source]+ (let [source-table-name (transforms-base.u/temp-table-name driver (namespace table-name))+ temp-table-name (u/poll {:thunk #(transforms-base.u/temp-table-name driver (namespace table-name))+ :done? #(not= source-table-name %)+ :interval-ms 1})]+ (log/info "Using rename-tables strategy with atomicity guarantees")+ (try+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)+ (transforms-base.u/rename-tables! driver db-id {table-name temp-table-name+ source-table-name table-name})+ (transforms-base.u/drop-table! driver db-id temp-table-name)+ (catch Exception e+ (log/error e "Failed to transfer data using rename-tables strategy")+ (try+ (transforms-base.u/drop-table! driver db-id source-table-name)+ (catch Exception _))+ (throw e)))))++(defn- transfer-with-create-drop-rename-strategy!+ "Transfer data using create + drop + rename to minimize time without data."+ [driver db-id table-name metadata data-source]+ (let [source-table-name (transforms-base.u/temp-table-name driver (namespace table-name))]+ (log/info "Using create-drop-rename strategy to minimize downtime")+ (try+ (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)+ (transforms-base.u/drop-table! driver db-id table-name)+ (driver/rename-table! driver db-id source-table-name table-name)+ (catch Exception e+ (log/error e "Failed to transfer data using create-drop-rename strategy")+ (try+ (transforms-base.u/drop-table! driver db-id source-table-name)+ (catch Exception _))+ (throw e)))))++(defn- transfer-with-drop-create-fallback-strategy!+ "Transfer data using drop + create fallback strategy."+ [driver db-id table-name metadata data-source]+ (log/info "Using drop-create fallback strategy")+ (try+ (transforms-base.u/drop-table! driver db-id table-name)+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)+ (catch Exception e+ (log/error e "Failed to transfer data using drop-create fallback strategy")+ (throw e))))++(defmulti ^:private transfer-file-to-db+ {:arglists '([driver db transform metadata temp-file])}+ (fn [_ _ transform _ _] (-> transform :target :type keyword)))++(defmethod transfer-file-to-db :table-incremental+ [driver {db-id :id}+ {:keys [target] :as transform}+ metadata temp-file]+ (let [table-name (transforms-base.u/qualified-table-name driver target)+ table-exists? (transforms-base.u/target-table-exists? transform)+ data-source {:type :jsonl-file+ :file temp-file}]+ (if (not table-exists?)+ (do+ (log/info "New table")+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))+ (insert-data! driver db-id (table-schema table-name metadata) data-source))))++(defmethod transfer-file-to-db :table+ [driver {db-id :id :as db}+ {:keys [target] :as transform}+ metadata temp-file]+ (let [table-name (transforms-base.u/qualified-table-name driver target)+ table-exists? (transforms-base.u/target-table-exists? transform)+ data-source {:type :jsonl-file+ :file temp-file}]+ (cond+ (not table-exists?)+ (do+ (log/info "New table")+ (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))++ (driver.u/supports? driver :atomic-renames db)+ (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)++ (driver.u/supports? driver :rename db)+ (transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)++ :else+ (transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))++;;; ------------------------------------------------- Cancellation -------------------------------------------------++(defn- start-cancellation-process!+ "Starts a core.async process that sends a cancellation request to the python executor if cancel-chan receives a value."+ [server-url run-id cancel-chan]+ (a/go (when (a/<! cancel-chan)+ (python-runner/cancel-python-code-http-call! server-url run-id))))++;;; ------------------------------------------------- Core Execution -------------------------------------------------++(defn- run-python-transform-impl!+ "Core Python transform execution. Returns {:status :result :logs :events}.++ Options:+ - `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"+ [{:keys [source] :as transform} db run-id cancel-chan message-log {:keys [with-stage-timing-fn]}]+ ;; Resolve name-based source table refs to table IDs (throws if any not found)+ (let [resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables source))]+ (with-open [shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]+ (let [driver (:engine db)+ server-url (transforms-python.settings/python-runner-url)+ _ (python-runner/copy-tables-to-s3! {:run-id run-id+ :shared-storage @shared-storage-ref+ :source (assoc source :source-tables resolved-source-tables)+ :cancel-chan cancel-chan+ :limit (:limit source)+ :transform-id (:id transform)})+ _ (start-cancellation-process! server-url run-id cancel-chan)+ {:keys [status body] :as response}+ (python-runner/execute-python-code-http-call!+ {:server-url server-url+ :code (:body source)+ :run-id run-id+ :table-name->id resolved-source-tables+ :shared-storage @shared-storage-ref})++ output-manifest (python-runner/read-output-manifest @shared-storage-ref)+ events (python-runner/read-events @shared-storage-ref)]+ (replace-python-logs! message-log events)+ (if (not= 200 status)+ (throw (ex-info "Python runner call failed"+ {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))+ :status-code 400+ :api-status-code status+ :body body+ :events events+ :timeout (:timeout body)}))+ (try+ (let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))+ temp-file ^File (.toFile temp-path)]+ (when-not (seq (:fields output-manifest))+ (throw (ex-info "No fields in metadata"+ {:metadata output-manifest+ :raw-body body+ :events events})))+ (try+ (with-open [in (python-runner/open-output @shared-storage-ref)]+ (io/copy in temp-file))+ ;; Transfer file to database with instrumentation+ (let [file-size (.length temp-file)+ do-transfer (fn [] (transfer-file-to-db driver db transform output-manifest temp-file))]+ (if with-stage-timing-fn+ (with-stage-timing-fn run-id [:import :file-to-dwh] do-transfer)+ (do-transfer))+ (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))+ (finally+ (.delete temp-file))))+ response+ (catch Exception e+ (log/error e "Failed to create resulting table")+ (throw (ex-info "Failed to create the resulting table"+ {:transform-message (or (:transform-message (ex-data e))+ (i18n/tru "Failed to create the resulting table"))}+ e)))))))))++;;; ------------------------------------------------- Base Execution Entry Point -------------------------------------------------++(defn run-python-transform!+ "Execute Python transform. Returns result map.++ Does:+ - Resolve source tables+ - Copy to S3+ - Call python runner+ - Transfer result to DB+ - Sync target++ Does NOT:+ - Create transform_run row+ - Poll/save logs to transform_run.message++ Options:+ - `:cancelled?` - (fn [] boolean), polled to check for cancellation+ - `:run-id` - optional, for cancellation signaling and instrumentation+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk] result)+ - `:message-log` - optional, pre-created message log atom (for scheduled execution log polling)+ - `:cancel-chan` - optional, pre-created promise-chan for cancellation (avoids bridging latency)++ Returns:+ {:status :succeeded | :failed | :cancelled+ :result <http response>+ :logs <string>+ :error <exception if failed>}"+ [transform {:keys [cancelled? run-id with-stage-timing-fn message-log cancel-chan]}]+ (assert (transforms-base.u/python-transform? transform) "Transform must be a python transform")+ (let [message-log (or message-log (empty-message-log))]+ (try+ ;; Check cancellation before starting+ (when (and cancelled? (cancelled?))+ (throw (ex-info "Transform cancelled before start" {:status :cancelled})))++ (let [{:keys [target] transform-id :id} transform+ db (t2/select-one :model/Database (:database target))+ ;; Use run-id if provided, otherwise generate a temp one for python runner+ effective-run-id (or run-id (rand-int Integer/MAX_VALUE))+ cancel-chan (or cancel-chan+ (let [ch (a/promise-chan)]+ ;; Bridge cancelled? to cancel-chan+ (when cancelled?+ (a/go+ (loop []+ (when-not (a/poll! ch)+ (if (cancelled?)+ (a/>! ch :cancel!)+ (do+ (a/<! (a/timeout 100))+ (recur)))))))+ ch))+ start-ms (u/start-timer)]++ (log! message-log (i18n/tru "Executing Python transform"))+ (log/info "Executing Python transform" transform-id "with target" (pr-str target))++ (let [result (run-python-transform-impl! transform db effective-run-id cancel-chan message-log+ {:with-stage-timing-fn with-stage-timing-fn})]+ (log! message-log (i18n/tru "Python execution finished successfully in {0}"+ (u.format/format-milliseconds (u/since-ms start-ms))))++ ;; Check cancellation after python+ (when (and cancelled? (cancelled?))+ (throw (ex-info "Transform cancelled after python execution" {:status :cancelled})))++ {:status :succeeded+ :result result+ :logs (message-log->string message-log)}))++ (catch Exception e+ (let [data (ex-data e)+ logs (message-log->string message-log)+ error-message (or (:transform-message data) (ex-message e))]+ (cond+ (= :cancelled (:status data))+ {:status :cancelled+ :error e+ :logs logs}++ (:timeout data)+ {:status :timeout+ :error e+ :logs (or (when-let [events (:events data)]+ (str/join "\n" (map :message events)))+ logs)}++ :else+ (do+ (log/error e "Error executing Python transform")+ {:status :failed+ :error e+ :logs (str logs "\n" error-message)})))))))++;;; ------------------------------------------------- Interface Implementation -------------------------------------------------++(defmethod transforms-base.i/execute-base! :python+ [transform opts]+ (run-python-transform! transform opts))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 30df0fa472b..1a88ad71a40 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -1,21 +1,16 @@(ns metabase-enterprise.transforms-python.execute(:require[clojure.core.async :as a]- [clojure.java.io :as io][clojure.string :as str]+ [metabase-enterprise.transforms-python.base :as base][metabase-enterprise.transforms-python.python-runner :as python-runner]- [metabase-enterprise.transforms-python.s3 :as s3]- [metabase-enterprise.transforms-python.settings :as transforms-python.settings][metabase.app-db.core :as app-db][metabase.driver :as driver][metabase.driver.connection :as driver.conn]- [metabase.driver.util :as driver.u]- [metabase.transforms.core :as transforms]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.instrumentation :as transforms.instrumentation]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.util :as transforms.util]- [metabase.util :as u]- [metabase.util.format :as u.format]+ [metabase.transforms.util :as transforms.u][metabase.util.i18n :as i18n][metabase.util.jvm :as u.jvm][metabase.util.log :as log]@@ -23,52 +18,16 @@(:import(java.io Closeable)(java.net SocketException)- (java.nio.file Files)- (java.nio.file.attribute FileAttribute)(java.time Duration)))(set! *warn-on-reflection* true)-(defn- empty-message-log- "Returns a new message log.-- Python transforms produce output while they run. The message log is a stateful structure that buffers log outputs during the run.- The goal is to assist in debugging in the case of python runtime or syntax errors, and to provide some immediate feedback for longer running transforms.-- Important Note: in memory buffering is considered acceptable as we expect /logs output to be truncated or small.-- The intended use of this structure is to produce a string suitable for `transform_run.message`, saving it every few seconds."- []- (atom {:pre-python [] ; log! outputs previous to the python execution, i.e table read progress- :python nil ; events json structured logs from the /logs endpoint- :post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target--(defn- log!- "Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."- [message-log s]- (swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))- nil)--(defn- replace-python-logs!- "Called when events output is received from a /logs call to copy those events into the message-log."- [message-log events]- (swap! message-log assoc :python events)- nil)--(defn- message-log->transform-run-message- "Presents the content of the log as a string suitable for storage in transform_run.message."- [message-log]- (let [{:keys [pre-python python post-python]} @message-log]- (str/join "\n" (concat pre-python- (map :message python)- post-python))))-(defn- save-log-to-transform-run-message!"Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."[run-id message-log](t2/update! :model/TransformRun:id run-id- {:message (message-log->transform-run-message message-log)}))+ {:message (base/message-log->string message-log)}))(def ^:private ^Duration python-message-loop-sleep-duration(Duration/ofMillis 1000))@@ -93,7 +52,7 @@(do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)(recur))(do- (replace-python-logs! message-log events)+ (swap! message-log assoc :python events)(save-log-to-transform-run-message! run-id message-log)(recur))))(= 404 status)@@ -110,213 +69,24 @@(log/errorf e "An exception was caught during msg update loop, run-id: %s" run-id))))(defn- open-python-message-update-future! ^Closeable [run-id message-log]- (let [cleanup (fn [fut]- (future-cancel fut)- (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))- (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)- (log/debugf "Log polling task done, run-id: %s" run-id)))- fut (u.jvm/in-virtual-thread*- (python-message-update-loop! run-id message-log))]- (reify Closeable- (close [_] (cleanup fut)))))--(defn- table-schema [table-name metadata]- {:name (if (keyword? table-name) table-name (keyword table-name))- :columns (mapv (fn [{:keys [name base_type #_database_type]}]- {:name name- :type (python-runner/restricted-insert-type base_type)- ;; :database-type database_type- :nullable? true})- (:fields metadata))})--(defn- insert-data!- "Insert data from source into an existing table."- [driver db-id table-schema data-source]- (let [data-source (assoc data-source :table-schema table-schema)]- (driver/insert-from-source! driver db-id table-schema data-source)))--(defn- create-table-and-insert-data!- "Create a table from metadata and insert data from source."- [driver db-id table-schema data-source]- (transforms.util/create-table-from-schema! driver db-id table-schema)- (insert-data! driver db-id table-schema data-source))--(defn- transfer-with-rename-tables-strategy!- "Transfer data using the rename-tables*! multimethod with atomicity guarantees.- Creates new table, then atomically renames target->old and new->target, then drops old."- [driver db-id table-name metadata data-source]- (let [source-table-name (driver.u/temp-table-name driver table-name)- temp-table-name (driver.u/temp-table-name driver table-name)]- (log/info "Using rename-tables strategy with atomicity guarantees")- (try-- (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)- (transforms.util/rename-tables! driver db-id {table-name temp-table-name- source-table-name table-name})- (transforms.util/drop-table! driver db-id temp-table-name)-- (catch Exception e- (log/error e "Failed to transfer data using rename-tables strategy")- (try- (transforms.util/drop-table! driver db-id source-table-name)- (catch Exception _))- (throw e)))))--(defn- transfer-with-create-drop-rename-strategy!- "Transfer data using create + drop + rename to minimize time without data.- Creates new table, drops old table, then renames new->target."- [driver db-id table-name metadata data-source]- (let [source-table-name (driver.u/temp-table-name driver table-name)]- (log/info "Using create-drop-rename strategy to minimize downtime")- (try-- (create-table-and-insert-data! driver db-id (table-schema source-table-name metadata) data-source)- (transforms.util/drop-table! driver db-id table-name)- (driver/rename-table! driver db-id source-table-name table-name)-- (catch Exception e- (log/error e "Failed to transfer data using create-drop-rename strategy")- (try- (transforms.util/drop-table! driver db-id source-table-name)- (catch Exception _))- (throw e)))))--(defn- transfer-with-drop-create-fallback-strategy!- "Transfer data using drop + create fallback strategy.- Drops old table, then creates new table directly with target name."- [driver db-id table-name metadata data-source]- (log/info "Using drop-create fallback strategy")- (try-- (transforms.util/drop-table! driver db-id table-name)- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source)-- (catch Exception e- (log/error e "Failed to transfer data using drop-create fallback strategy")- (throw e))))--(defmulti ^:private transfer-file-to-db- {:arglists '([driver db transform metadata temp-file])}- (fn [_ _ transform _ _] (-> transform :target :type keyword)))--(defmethod transfer-file-to-db :table-incremental- [driver {db-id :id}- {:keys [target] :as transform}- metadata temp-file]- (let [table-name (transforms.util/qualified-table-name driver target)- table-exists? (transforms.util/target-table-exists? transform)- data-source {:type :jsonl-file- :file temp-file}]-- ;; once we have more than just append, dispatch on :target-incremental-strategy-- (if (not table-exists?)- (do- (log/info "New table")- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))- (insert-data! driver db-id (table-schema table-name metadata) data-source))))--(defmethod transfer-file-to-db :table- [driver {db-id :id :as db}- {:keys [target] :as transform}- metadata temp-file]- (let [table-name (transforms.util/qualified-table-name driver target)- table-exists? (transforms.util/target-table-exists? transform)- data-source {:type :jsonl-file- :file temp-file}]- (cond- (not table-exists?)- (do- (log/info "New table")- (create-table-and-insert-data! driver db-id (table-schema table-name metadata) data-source))-- (driver.u/supports? driver :atomic-renames db)- (transfer-with-rename-tables-strategy! driver db-id table-name metadata data-source)-- (driver.u/supports? driver :rename db)- (transfer-with-create-drop-rename-strategy! driver db-id table-name metadata data-source)-- :else- (transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata data-source))))--(defn- start-cancellation-process!- "Starts a core.async process that optimistically sends a cancellation request to the python executor if cancel-chan receives a value.- Returns a channel that will receive either the async http call j.u.c.FutureTask in the case of cancellation, or nil when the cancel-chan is closed."- [server-url run-id cancel-chan]- (a/go (when (a/<! cancel-chan)- (python-runner/cancel-python-code-http-call! server-url run-id))))--(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]- ;; Resolve name-based source table refs to table IDs (throws if any not found)- (let [resolved-source-tables (transforms.util/resolve-source-tables (:source-tables source))]- ;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms- (with-open [^Closeable log-future-ref- (if (app-db/in-transaction?)- ;; if in a transaction (such as under mt/with-temp), it is not safe to poll for logs (close race / contention)- ;; tests that want to test async log behaviour should opt out of thread-local test helpers- (reify Closeable (close [_]))- (open-python-message-update-future! run-id message-log))- shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]- (let [driver (:engine db)- server-url (transforms-python.settings/python-runner-url)- _ (python-runner/copy-tables-to-s3! {:run-id run-id- :shared-storage @shared-storage-ref- :source (assoc source :source-tables resolved-source-tables)- :cancel-chan cancel-chan- :limit (:limit source)- :transform-id (:id transform)})- _ (start-cancellation-process! server-url run-id cancel-chan) ; inherits lifetime of cancel-chan- {:keys [status body] :as response}- (python-runner/execute-python-code-http-call!- {:server-url server-url- :code (:body source)- :run-id run-id- :table-name->id resolved-source-tables- :shared-storage @shared-storage-ref})-- output-manifest (python-runner/read-output-manifest @shared-storage-ref)- events (python-runner/read-events @shared-storage-ref)]- (.close ^Closeable log-future-ref) ; early close to force any writes to flush- (replace-python-logs! message-log events)- (if (not= 200 status)- (do- (when (:timeout body)- (transforms/timeout-run! run-id))- (throw (ex-info "Python runner call failed"- {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))- :status-code 400- :api-status-code status- :body body- :events events})))- (try- (let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))- temp-file (.toFile temp-path)]- (when-not (seq (:fields output-manifest))- (throw (ex-info "No fields in metadata"- {:metadata output-manifest- :raw-body body- :events events})))- (try- (with-open [in (python-runner/open-output @shared-storage-ref)]- (io/copy in temp-file))- (let [file-size (.length temp-file)]- (transforms.instrumentation/with-stage-timing [run-id [:import :file-to-dwh]]- (transfer-file-to-db driver db transform output-manifest temp-file))- (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))- (finally- (.delete temp-file))))- response- (catch Exception e- (log/error e "Failed to to create resulting table")- (throw (ex-info "Failed to create the resulting table"- {:transform-message (or (:transform-message (ex-data e))- ;; TODO keeping messaging the same at this level- ;; should be more specific in underlying calls- (i18n/tru "Failed to create the resulting table"))}- e)))))))))+ (if (app-db/in-transaction?)+ ;; in a transaction (such as under mt/with-temp), it is not safe to poll for logs+ (do+ (log/warnf "Log polling disabled as we are within a database transaction, run-id: %s" run-id)+ (reify Closeable (close [_])))+ ;; poll updates until execution is complete+ (let [cleanup (fn [fut]+ (future-cancel fut)+ (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))+ (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)+ (log/debugf "Log polling task done, run-id: %s" run-id)))+ fut (u.jvm/in-virtual-thread*+ (python-message-update-loop! run-id message-log))]+ (reify Closeable+ (close [_] (cleanup fut))))))(defn- exceptional-run-message [message-log ex]- (str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)+ (str/join "\n" (remove str/blank? [(base/message-log->string message-log)(or (:transform-message (ex-data ex))(if (instance? InterruptedException ex)(i18n/tru "Transform interrupted")@@ -327,43 +97,43 @@Blocks until the transform returns."[transform {:keys [run-method start-promise user-id]}]- (assert (transforms.util/python-transform? transform) "Transform must be a python transform")(try- (let [message-log (empty-message-log)+ (let [message-log (base/empty-message-log){:keys [target owner_user_id creator_id] transform-id :id} transform- {driver :engine :as db} (t2/select-one :model/Database (transforms.i/target-db-id transform))- ;; For manual runs, use the triggering user; for cron, use owner/creator+ {driver :engine :as db} (t2/select-one :model/Database (transforms-base.i/target-db-id transform))run-user-id (if (and (= run-method :manual) user-id)user-id(or owner_user_id creator_id))- {run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method run-user-id)]+ {run-id :id} (transforms.u/try-start-unless-already-running transform-id run-method run-user-id)](some-> start-promise (deliver [:started run-id]))- (log! message-log (i18n/tru "Executing Python transform"))- (driver.conn/with-write-connection- (log/info "Executing Python transform" transform-id "with target" (pr-str target)- (when (driver.conn/write-connection-requested?)- " using write connection"))- (let [start-ms (u/start-timer)- conn-spec (driver/connection-spec driver db)- transform-details {:db-id (:id db)- :transform-id transform-id- :transform-type (keyword (:type target))- :conn-spec conn-spec- :output-schema (:schema target)- :output-table (transforms.util/qualified-table-name driver target)}- run-fn (fn [cancel-chan]- (run-python-transform! transform db run-id cancel-chan message-log)- (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))- (save-log-to-transform-run-message! run-id message-log))- ex-message-fn #(exceptional-run-message message-log %)- result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]- (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]- (transforms.util/handle-transform-complete!- :run-id run-id- :transform transform- :db db)- {:run_id run-id- :result result})))+ (with-open [_ (open-python-message-update-future! run-id message-log)]+ (driver.conn/with-write-connection+ (let [conn-spec (driver/connection-spec driver db)+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}+ run-fn (fn [cancel-chan]+ (let [result (transforms-base.i/execute-base!+ transform+ {:cancelled? #(boolean (a/poll! cancel-chan))+ :cancel-chan cancel-chan+ :run-id run-id+ :message-log message-log+ :with-stage-timing-fn (fn [rid stage thunk]+ (transforms.instrumentation/with-stage-timing [rid stage]+ (thunk)))})]+ (save-log-to-transform-run-message! run-id message-log)+ (when-not (= :succeeded (:status result))+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))+ result))+ ex-message-fn #(exceptional-run-message message-log %)+ result (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]+ (transforms.u/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn))]+ ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)+ (transforms-base.u/complete-execution! transform+ {:run-id run-id+ :with-stage-timing-fn (fn [rid stage thunk]+ (transforms.instrumentation/with-stage-timing [rid stage]+ (thunk)))})+ {:run_id run-id :result result}))))(catch Throwable t(log/error t "Error executing Python transform")(throw t))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.cljindex 64cf330c203..bb23a6940ab 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/impl.clj@@ -3,37 +3,7 @@[metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase.transforms.interface :as transforms.i]))-(defmethod transforms.i/target-db-id :python- [transform]- (-> transform :target :database))--(defmethod transforms.i/source-db-id :python- [transform]- (-> transform :source :source-database))-#_{:clj-kondo/ignore [:discouraged-var]}(defmethod transforms.i/execute! :python[transform options](transforms-python.execute/execute-python-transform! transform options))--(defn- source-table-value->dependency- "Convert a source table value (int or ref map) to a dependency map."- [v]- (cond- ;; Integer table ID - direct table dependency- (int? v)- {:table v}-- ;; Map with resolved table_id - direct table dependency- (:table_id v)- {:table (:table_id v)}-- ;; Map without table_id - return ref for ordering system to resolve- :else- {:table-ref (select-keys v [:database_id :schema :table])}))--(defmethod transforms.i/table-dependencies :python- [transform]- (into #{}- (map source-table-value->dependency)- (vals (get-in transform [:source :source-tables]))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 9432ddc6960..49691b82aa6 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -14,8 +14,8 @@[metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp][metabase.query-processor.pipeline :as qp.pipeline]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.instrumentation :as transforms.instrumentation]- [metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.json :as json]@@ -277,7 +277,7 @@transform (t2/select-one :model/Transform transform-id)](cond-> (-> (lib/query metadata-provider table-metadata)lib/disable-default-limit)- source-incremental-strategy (transforms.u/preprocess-incremental-query source-incremental-strategy (transforms.u/next-checkpoint transform))+ source-incremental-strategy (transforms-base.u/preprocess-incremental-query source-incremental-strategy (transforms-base.u/next-checkpoint transform))limit (lib/limit limit))));; TODO break this up such that s3 can be swapped out for other transfer mechanisms.@@ -294,6 +294,7 @@(> (count (:source-tables source)) 1))(throw (ex-info "Incremental transforms for python only supports one source table" {})))(doseq [[table-name v] (:source-tables source)+ ;; TODO (Chris 2026-03-03) We should also handle the case where table-id has not been backfilled yet...:let [table-id (if (int? v) v (:table_id v)){:keys [s3-client bucket-name objects]} shared-storage{data-path :path} (get objects [:table table-id :data])diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj b/enterprise/backend/src/metabase_enterprise/workspaces/api.cljindex 557dd408005..6df753785a3 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/api.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/api.clj@@ -29,7 +29,7 @@[metabase.request.core :as request][metabase.transforms.core :as transforms][metabase.transforms.feature-gating :as transforms.gating]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.i18n :as i18n :refer [deferred-tru tru]][metabase.util.malli.registry :as mr]@@ -79,7 +79,7 @@[:python[:map {:closed true}[:source-database {:optional true} :int]- [:source-tables [:map-of [:string {:min 1}] :int]]+ [:source-tables [:map-of [:string {:min 1}] [:or :int :map]]][:type [:= "python"]][:body :string]]]])@@ -801,7 +801,7 @@(doseq [field [:name :source :target]](api/check-400 (get body field) (str (name field) " is required when creating a new transform")));; Check premium feature requirements- (api/check (transforms.util/check-feature-enabled body)+ (api/check (transforms.u/check-feature-enabled body)[402 (deferred-tru "Premium features required for this transform type are not enabled.")])(t2/with-transaction [_tx](let [workspace (u/prog1 (api/check-404 (t2/select-one :model/Workspace :id ws-id))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj b/enterprise/backend/src/metabase_enterprise/workspaces/common.cljindex 9a7d4eddeb2..6df7b59b6f0 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/common.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/common.clj@@ -6,7 +6,7 @@[metabase-enterprise.workspaces.util :as ws.u][metabase.api-keys.core :as api-key][metabase.api.common :as api]- [metabase.transforms.interface :as transforms.i]+ [metabase.transforms-base.interface :as transforms-base.i][metabase.util.log :as log][metabase.util.quick-task :as quick-task][toucan2.core :as t2]))@@ -157,7 +157,7 @@(ws.u/assert-transform! entity-type);; Initialize workspace if uninitialized (outside transaction so async task can see committed data)(let [workspace (if (= :uninitialized (:db_status workspace))- (let [target-db-id (transforms.i/target-db-id body)]+ (let [target-db-id (transforms-base.i/target-db-id body)](api/check-400 target-db-id "Transform must have a target database")(initialize-workspace! workspace target-db-id))workspace)]diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.cljindex 5066efea514..c97c323abc1 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/dependencies.clj@@ -175,13 +175,25 @@(inputs-from-native-query query db-id)(inputs-from-mbql-query query)))+(defn- source-table-ref->table-ref+ "Paper over a small unfortunate difference in shape"+ [table-ref]+ (-> (select-keys table-ref [:schema :table :table_id])+ (assoc :db_id (:database_id table-ref))))+(defn- inputs-from-python-transform"Extract table refs from a python transform's source-tables.- Python transforms require tables to exist (they map name -> table_id). Batch lookup."+ Uses metadata from map refs directly; only looks up the database for bare integer IDs."[source-tables]- (let [table-ids (set (vals source-tables))- table-refs (batch-table-refs-from-ids table-ids)]- (u/keepv table-refs (vals source-tables))))+ (let [{maps true ints false} (group-by (comp map? val) source-tables)+ ;; Maps already have schema/table metadata+ map-refs (map (comp source-table-ref->table-ref val) maps)+ ;; Integers need a batch lookup+ int-ids (map val ints)+ int-refs (when (seq int-ids)+ (let [lookup (batch-table-refs-from-ids int-ids)]+ (keep lookup int-ids)))]+ (into (vec map-refs) int-refs)))(mu/defn analyze-entity :- ::analysis"Analyze a workspace entity to find its dependencies.diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj b/enterprise/backend/src/metabase_enterprise/workspaces/execute.cljindex cce652ab593..ac636ef9e8f 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/execute.clj@@ -1,37 +1,20 @@(ns metabase-enterprise.workspaces.execute- "This namespace is concerned with executing non-AppDB transforms, with optional reference re-mappings.+ "Executing transforms with optional reference re-mappings, without writing to AppDB.- It currently lives inside the workspace module, but eventually will become part of the transform module, so it should- not make any assumptions about being run within a workspace, having the input / output semantics of a workspace, and- especially it should not touch any of the Workspace entities inside AppDb, or any of the other workspace namespaces.-- For now, it uses AppDB as a side-channel with the transforms module, but in future this module should be COMPLETELY- decoupled from AppDb."+ Uses transforms-base.core/execute! to run transforms in-memory. No Transform or TransformRun+ rows are created in AppDB. Warehouse DB changes (actual table data) persist in the isolated schema."(:require[clojure.string :as str][metabase-enterprise.transforms-python.python-runner :as python-runner]- [metabase.api.common :as api][metabase.query-processor :as qp][metabase.sql-tools.core :as sql-tools]- [metabase.transforms.core :as transforms]- [metabase.transforms.execute :as transforms.execute]- [metabase.transforms.util :as transforms.util]- [metabase.util :as u]+ [metabase.transforms-base.core :as transforms-base]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util.log :as log][toucan2.core :as t2]))(set! *warn-on-reflection* true)-(defn- execution-results- "Extract execution metadata from transform_run and target table info.- Must be called within the transaction before rollback."- [{xf-id :id :keys [target]}]- (let [run (t2/select-one :model/TransformRun :transform_id xf-id)]- (merge- (select-keys run [:status :start_time :end_time :message])- {:table {:name (:name target)- :schema (:schema target)}})))-(defn- remap-python-source"Remap source-tables in a Python transform source to point to isolated tables.Handles both legacy integer format and new map format (from PR #66934)."@@ -113,7 +96,7 @@[{:keys [source]} remapping](let [table-mapping (:tables remapping no-mapping)remapped-source (remap-python-source table-mapping source)- resolved-source-tables (transforms.util/resolve-source-tables (:source-tables remapped-source))+ resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables remapped-source)){:as result:keys [cols rows]} (python-runner/execute-and-read-output!{:code (:body remapped-source)@@ -161,7 +144,7 @@"Run SQL transform query and return first 2000 rows without persisting.Returns a ::ws.t/query-result map with data nested under :data to match /api/dataset format."[{:keys [source]} remapping]- (run-query source (transforms/transform-source-type source) remapping+ (run-query source (transforms-base.u/transform-source-type source) remapping{:error-context "Failed to run sql dry-run"}))(defn execute-adhoc-sql@@ -182,46 +165,35 @@"Execute transform and return first 2000 rows without persisting.Returns a ::ws.t/query-result map with data nested under :data."[{:keys [source] :as transform} remapping]- (let [s-type (transforms/transform-source-type source)]+ (let [s-type (transforms-base.u/transform-source-type source)](case s-type(:native :mbql) (dry-run-sql transform remapping):python (dry-run-python transform remapping))))(defn run-transform-with-remapping- "Execute a given collection with the given table and field re-mappings.+ "Execute a given transform with the given table and field re-mappings.This is used by Workspaces to re-route the output of each transform to a non-production table, andto re-write their queries where these outputs transitively become inputs to other transforms.Returns an ::ws.t/execution-result map with status, timing, and table metadata.- --------- NOTE: currently execution is done using transaction-rollback pattern, as a short-term hack.-- 1. Creating temporary Transform/TransformRun records in a transaction- 2. Executing using existing transform infrastructure- 3. Scraping metadata (execution stats + table schema)- 4. Rolling back the transaction (no app DB records persist)-- The warehouse DB changes (actual table data) DO persist in the isolated schema."+ Uses transforms-base.core/execute! to run the transform in-memory without writing+ Transform or TransformRun rows to AppDB. The warehouse DB changes (actual table data)+ DO persist in the isolated schema."[{:keys [source target] :as transform} remapping]- (t2/with-transaction [_conn]- (let [s-type (transforms/transform-source-type source)- table-mapping (:tables remapping no-mapping)- target-fallback (:target-fallback remapping no-mapping)- field-mapping (:fields remapping no-mapping)- new-xf (-> (select-keys transform [:name :description])- (assoc :creator_id api/*current-user-id*- :source (remap-source table-mapping field-mapping s-type source)- :target (remap-target table-mapping target-fallback target)))- _ (assert (:target new-xf) "Target mapping must not be nil")- temp-xf (t2/insert-returning-instance! :model/Transform new-xf)]- (try- (transforms.execute/execute! temp-xf {:run-method :manual})- (catch Exception _e- ;; Execution failed - the TransformRun record has been updated with failure status and message- nil))- ;; Return execution results whether succeeded or failed- (u/prog1 (execution-results temp-xf)- (t2/delete! :model/Transform (:id temp-xf))))))+ (let [s-type (transforms-base.u/transform-source-type source)+ table-mapping (:tables remapping no-mapping)+ target-fallback (:target-fallback remapping no-mapping)+ field-mapping (:fields remapping no-mapping)+ remapped-xf (-> (select-keys transform [:name :description])+ (assoc :source (remap-source table-mapping field-mapping s-type source)+ :target (remap-target table-mapping target-fallback target)))+ _ (assert (:target remapped-xf) "Target mapping must not be nil")+ start-time (java.time.Instant/now)+ result (transforms-base/execute! remapped-xf {:publish-events? false})]+ {:status (:status result)+ :start_time start-time+ :end_time (java.time.Instant/now)+ :message (or (:logs result) (some-> (:error result) ex-message))+ :table (select-keys (:target remapped-xf) [:name :schema])}))diff --git a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.clj b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.cljindex 59a72030f4b..58b66c28224 100644--- a/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.clj+++ b/enterprise/backend/src/metabase_enterprise/workspaces/models/workspace_transform.clj@@ -3,10 +3,8 @@and edited within a workspace."(:require[clojure.string :as str]- [medley.core :as m]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib][metabase.models.interface :as mi]+ [metabase.models.transforms.transform :as transform][methodical.core :as methodical][toucan2.core :as t2]))@@ -24,25 +22,9 @@(derive :hook/timestamped?)(derive :hook/entity-id))-;; TODO (Chris 2025-12-11) -- we need to share a bunch of stuff with transforms, i think we'll need to reorganize modules-;; suggestion: add a transforms-interfaces module which both transforms and workspaces depend on.--(defn- transform-source-out-DUPLICATED [m]- (-> m- mi/json-out-without-keywordization- (update-keys keyword)- (m/update-existing :query lib-be/normalize-query)- (m/update-existing :type keyword)- (m/update-existing :source-incremental-strategy #(update-keys % keyword))))--(defn- transform-source-in-DUPLICATED [m]- (-> m- (m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))- mi/json-in))-(t2/deftransforms :model/WorkspaceTransform{:ref_id {:in identity :out str/trim}- :source {:out transform-source-out-DUPLICATED, :in transform-source-in-DUPLICATED}+ :source {:out transform/transform-source-out, :in transform/transform-source-in}:target mi/transform-json})(t2/define-before-insert :model/WorkspaceTransformdiff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex 550aa9bae4d..a02eb8d6dcb 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -15,10 +15,10 @@[metabase.lib.metadata :as lib.metadata][metabase.query-processor :as qp][metabase.test :as mt]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.execute :as transforms.execute][metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]- [metabase.transforms.util :as transforms.u][next.jdbc :as next.jdbc][next.jdbc.result-set :as jdbc.rs][toucan2.core :as t2]))@@ -183,8 +183,8 @@(some-> result :data :rows first first bigint)))(defn get-checkpoint-value [transform]- (#'transforms.u/next-checkpoint-value- (transforms.u/next-checkpoint transform)))+ (#'transforms-base.u/next-checkpoint-value+ (transforms-base.u/next-checkpoint transform)))(defn- compare-checkpoint-values"Compare two checkpoint values with type-appropriate logic. "@@ -609,7 +609,7 @@(testing "sync has picked up table"(is (=? {:name target-table, :fields [{:name "id"}]} (-> (t2/select-one :model/Table :name target-table) (t2/hydrate :fields)))))(testing "checkpoint is recognized"- (is (some? (transforms.u/next-checkpoint transform))))))))))+ (is (some? (transforms-base.u/next-checkpoint transform))))))))))(deftest checkpoint-field-does-not-exist-test(mt/test-drivers #{:postgres} ; no db specifics@@ -636,7 +636,7 @@(testing "target table has expected data"(is (= [{:id 42}] (pg-table-rows db-spec target-table))))(testing "checkpoint is not recognized, so transform acts as if no checkpoint"- (is (nil? (transforms.u/next-checkpoint transform))))+ (is (nil? (transforms-base.u/next-checkpoint transform))));; Maybe this is unrealistic - you cannot select a column that does not exist;; But the source tables schema can change, e.g. rename: you change event_time to event_ts or something;; so one would have to be careful to disable or delete transforms ahead of a schema change like thisdiff --git a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.cljindex 2fc119cbb36..f6a81f187a7 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/ordering_test.clj@@ -2,8 +2,8 @@(:require[clojure.test :refer [deftest testing is]][metabase.test :as mt]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.ordering :as ordering]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.ordering :as ordering][toucan2.core :as t2]))(defn- make-python-transform@@ -34,7 +34,7 @@:table "intermediate_output"}}"final_output")](testing "table-dependencies returns table-ref for unresolved name reference"- (let [deps (transforms.i/table-dependencies (t2/select-one :model/Transform :id t-b))]+ (let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))](is (contains? deps {:table-ref {:database_id (mt/id):schema "public":table "intermediate_output"}}))))@@ -58,7 +58,7 @@:table "output_a"}}"output_b")](testing "table-dependencies includes both types"- (let [deps (transforms.i/table-dependencies (t2/select-one :model/Transform :id t-b))]+ (let [deps (transforms-base.i/table-dependencies (t2/select-one :model/Transform :id t-b))](is (contains? deps {:table (mt/id :products)}))(is (contains? deps {:table-ref {:database_id (mt/id):schema "public"diff --git a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/util_test.cljindex dde770706d3..27d8ece4dca 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/util_test.clj@@ -3,7 +3,7 @@[clojure.test :refer [deftest testing is]][metabase.driver.util :as driver.u][metabase.test :as mt]- [metabase.transforms.util :as transforms.util]))+ [metabase.transforms.util :as transforms.u]))(set! *warn-on-reflection* true)@@ -12,16 +12,16 @@(let [table-with-schema {:name (name (driver.u/temp-table-name :postgres :schema/orders))}table-without-schema {:name (name (driver.u/temp-table-name :postgres :orders))}](mt/with-premium-features #{:hosting}- (is (false? (transforms.util/is-temp-transform-table? table-with-schema)))- (is (false? (transforms.util/is-temp-transform-table? table-without-schema))))+ (is (false? (transforms.u/is-temp-transform-table? table-with-schema)))+ (is (false? (transforms.u/is-temp-transform-table? table-without-schema))))(mt/with-premium-features #{}- (is (transforms.util/is-temp-transform-table? table-without-schema))- (is (transforms.util/is-temp-transform-table? table-with-schema)))+ (is (transforms.u/is-temp-transform-table? table-without-schema))+ (is (transforms.u/is-temp-transform-table? table-with-schema)))(mt/with-premium-features #{:hosting :transforms}- (is (transforms.util/is-temp-transform-table? table-without-schema))- (is (transforms.util/is-temp-transform-table? table-with-schema)))))+ (is (transforms.u/is-temp-transform-table? table-without-schema))+ (is (transforms.u/is-temp-transform-table? table-with-schema)))))(testing "Ignores non-transform tables"(mt/with-premium-features #{:transforms}- (is (false? (transforms.util/is-temp-transform-table? {:name :orders})))- (is (false? (transforms.util/is-temp-transform-table? {:name :public/orders}))))))+ (is (false? (transforms.u/is-temp-transform-table? {:name :orders})))+ (is (false? (transforms.u/is-temp-transform-table? {:name :public/orders}))))))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.cljindex ba7bce4dd3c..606a8789afe 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/drivers_test.clj@@ -12,8 +12,8 @@[metabase.test :as mt][metabase.test.data.interface :as tx][metabase.test.data.sql :as sql.tx]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]- [metabase.transforms.util :as transforms.util][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.json :as json]@@ -121,7 +121,7 @@table-schema {:name qualified-table-name:columns columns}](mt/as-admin- (transforms.util/create-table-from-schema! driver db-id table-schema))+ (transforms-base.u/create-table-from-schema! driver db-id table-schema))(when (seq data)(driver/insert-from-source! driver db-id table-schemadiff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.cljindex fd97387be84..4fe6138a118 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj@@ -4,9 +4,10 @@[metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase.test :as mt][metabase.test.util :as test.util]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :as transforms.tu :refer [with-transform-cleanup!]]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms.util :as transforms.u][toucan2.core :as t2])(:import(java.util.concurrent CountDownLatch)))@@ -48,10 +49,10 @@" return pd.DataFrame({'name': ['Charlie', 'Diana', 'Eve'], 'age': [35, 40, 45]})")}}))(let [swap-latch (CountDownLatch. 1)- original-rename-tables-atomic! transforms.util/rename-tables!]- (with-redefs [transforms.util/rename-tables! (fn [driver db-id rename-pairs]- (.await swap-latch)- (original-rename-tables-atomic! driver db-id rename-pairs))]+ original-rename-tables-atomic! transforms-base.u/rename-tables!]+ (mt/with-dynamic-fn-redefs [transforms-base.u/rename-tables! (fn [driver db-id rename-pairs]+ (.await swap-latch)+ (original-rename-tables-atomic! driver db-id rename-pairs))](let [transform-future (future(transforms-python.execute/execute-python-transform!(t2/select-one :model/Transform (:id transform))@@ -91,7 +92,7 @@(let [db-id (mt/id)tables (t2/select :model/Table :db_id db-id :active true)]- (is (not-any? transforms.util/is-temp-transform-table? tables)+ (is (not-any? transforms.u/is-temp-transform-table? tables)"No temp tables should remain after successful Python transform")(is (= [[1 "a"] [2 "b"] [3 "c"]] (transforms.tu/table-rows table-name))diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.cljindex 177bc41dee5..af3c34383e4 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/job_test.clj@@ -59,9 +59,9 @@:name "Test Python Transform"}run-id 101logged-messages (atom [])]- (with-redefs [log/log* (fn [_ level _ message]- (swap! logged-messages conj {:level level :message message}))- transform-run/running-run-for-transform-id (constantly nil)]+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]+ (swap! logged-messages conj {:level level :message message}))+ transform-run/running-run-for-transform-id (constantly nil)](#'jobs/run-transform! run-id :scheduled nil python-transform)(is (= 1 (count @logged-messages))"Should log exactly one warning")diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.cljindex 785d3639815..441ef4ab21a 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/ordering_test.clj@@ -2,8 +2,8 @@(:require[clojure.test :refer :all][metabase.test :as mt]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.ordering :as ordering]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.ordering :as ordering][metabase.transforms.test-util :as transforms.tu][toucan2.core :as t2]))@@ -33,7 +33,7 @@(defn- transform-deps-for-db [transform](mt/with-metadata-provider (mt/id)- (#'transforms.i/table-dependencies transform)))+ (transforms-base.i/table-dependencies transform)))(deftest python-transform-basic-dependencies-test(testing "Python transforms with source-tables dependencies are extracted correctly"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.cljindex 5b1fc2ae7d5..e6532cb20a8 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/python_runner_test.clj@@ -15,7 +15,7 @@[metabase.test.data.interface :as tx][metabase.test.data.sql :as sql.tx][metabase.test.util :as tu]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util :as u][metabase.util.json :as json][toucan2.core :as t2]))@@ -395,11 +395,11 @@(deftest transform-function-without-libraries-test(testing "transform function works when no libraries exist"(mt/test-drivers #{:postgres}- (with-redefs [t2/select-fn->fn (fn [k v model]- (when (and (= k :path)- (= v :source)- (= model :model/PythonLibrary))- {}))]+ (mt/with-dynamic-fn-redefs [t2/select-fn->fn (fn [k v model]+ (when (and (= k :path)+ (= v :source)+ (= model :model/PythonLibrary))+ {}))](let [transform-code (str "import pandas as pd\n""\n""def transform():\n"@@ -412,11 +412,11 @@(deftest transform-function-library-import-error-test(testing "transform function handles missing library gracefully"(mt/test-drivers #{:postgres}- (with-redefs [t2/select-fn->fn (fn [k v model]- (when (and (= k :path)- (= v :source)- (= model :model/PythonLibrary))- {"utils" "def helper():\n return 42"}))]+ (mt/with-dynamic-fn-redefs [t2/select-fn->fn (fn [k v model]+ (when (and (= k :path)+ (= v :source)+ (= model :model/PythonLibrary))+ {"utils" "def helper():\n return 42"}))](let [transform-code (str "import pandas as pd\n""from common import some_function # This library doesn't exist\n""\n"@@ -452,7 +452,7 @@{:name "created_date" :type :type/Date :nullable? true}{:name "description" :type :type/Text :nullable? true}]}_ (mt/as-admin- (transforms.util/create-table-from-schema! driver db-id table-schema))+ (transforms-base.u/create-table-from-schema! driver db-id table-schema))row-values [[119.99diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.cljindex 23fbf41e020..a5cba2543be 100644--- a/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/transforms_api_test.clj@@ -2,6 +2,7 @@(:require[clojure.string :as str][clojure.test :refer :all]+ [metabase-enterprise.transforms-python.base :as transforms-python.base][metabase-enterprise.transforms-python.execute :as transforms-python.execute][metabase-enterprise.transforms-python.init][metabase-enterprise.transforms-python.python-runner :as transforms-python.python-runner]@@ -237,9 +238,9 @@(run-scenario [scenario schema](with-redefs [transforms-python.execute/python-message-loop-sleep-duration (Duration/ofMillis fast-log-polling-ms)- transforms-python.execute/transfer-file-to-db (if-some [e (:writeback-ex scenario)]+ transforms-python.base/transfer-file-to-db (if-some [e (:writeback-ex scenario)](fn [& _] (throw e))- @#'transforms-python.execute/transfer-file-to-db)]+ @#'transforms-python.base/transfer-file-to-db)](with-transform-cleanup! [target {:type "table":schema schema:name "result"}]@@ -383,7 +384,7 @@(fn [os fields-meta col-meta](rf-proxy ready-signal wait-signal (f os fields-meta col-meta)))}):write- (let [f-ref #'transforms-python.execute/transfer-file-to-db+ (let [f-ref #'transforms-python.base/transfer-file-to-dbf @f-ref]{f-ref(fn [& args]diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.cljindex a056b3214ae..943f05ef1b4 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/api_test.clj@@ -316,12 +316,12 @@(testing "No updates are propagated back to core app on merge failure"(let [update-transform! transforms.core/update-transform!]- (with-redefs [transforms.core/update-transform! (let [call-count (atom 0)]- (fn [& args]- (when (> @call-count 0)- (throw (Exception. "boom")))- (swap! call-count inc)- (apply update-transform! args)))]+ (mt/with-dynamic-fn-redefs [transforms.core/update-transform! (let [call-count (atom 0)]+ (fn [& args]+ (when (> @call-count 0)+ (throw (Exception. "boom")))+ (swap! call-count inc)+ (apply update-transform! args)))](testing "API response: empty merged, single error"(let [resp (mt/user-http-request :crowberto :post 200 (ws-url ws-id "/merge"))](is (empty? (get-in resp [:merged :transforms])))@@ -574,8 +574,8 @@_ (mt/user-http-request :crowberto :post 204(ws-url ws-id "/transform" ws-x-1-id "/archive"))](testing "Failure on merge"- (with-redefs [transforms.core/delete-transform! (fn [& _args]- (throw (Exception. "boom")))]+ (mt/with-dynamic-fn-redefs [transforms.core/delete-transform! (fn [& _args]+ (throw (Exception. "boom")))](let [resp (mt/user-http-request :crowberto :post 500(ws-url ws-id "/transform" ws-x-1-id "/merge"))](testing "Response"diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.cljindex 591602563e8..7d105fdcf60 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/execute_test.clj@@ -55,9 +55,8 @@:isolated_table_id number?}](t2/select :model/WorkspaceOutput :workspace_id (:id workspace) :ref_id (:ref_id ws-transform))))))- (testing "app DB records are rolled back"- ;; TransformRun is +1 because cascade delete was removed in d1e940e66b5- (is (= (update before :xfrun inc)+ (testing "no app DB records are created (transforms-base/execute! skips TransformRun)"+ (is (= before{:xf (t2/count :model/Transform):xfrun (t2/count :model/TransformRun)})))))))diff --git a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.tsindex 020affdbe2a..ae8b6e10a2e 100644--- a/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts+++ b/enterprise/frontend/src/metabase-enterprise/data-studio/workspaces/components/MergeWorkspaceModal/hooks/useTransformSources.ts@@ -3,6 +3,7 @@ import { useMemo } from "react";import { useGetTransformQuery } from "metabase/api";import { useSelector } from "metabase/lib/redux";import { getMetadata } from "metabase/selectors/metadata";+import { collapseSourceTableReferences } from "metabase/transforms/utils";import { useGetWorkspaceTransformQuery } from "metabase-enterprise/api";import type {PythonTransformSource,@@ -78,12 +79,14 @@ export function useTransformSources(return {oldSource,oldSourceTables: isPythonTransformSource(globalTransform?.source)- ? globalTransform.source["source-tables"]+ ? collapseSourceTableReferences(globalTransform.source["source-tables"]): undefined,oldTarget: globalTransform?.target,newSource,newSourceTables: isPythonTransformSource(workspaceTransform?.source)- ? workspaceTransform.source["source-tables"]+ ? collapseSourceTableReferences(+ workspaceTransform.source["source-tables"],+ ): undefined,newTarget: workspaceTransform?.target,hasError,diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsxindex 5b252faeda2..71f035beee5 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/PythonDataPicker.tsx@@ -1,10 +1,12 @@-import { useEffect, useState } from "react";+import { useEffect, useMemo, useState } from "react";import { t } from "ttag";import { skipToken, useListTablesQuery } from "metabase/api";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";+import { extractTableId } from "metabase/transforms/utils";import { Box, Button, Icon, Stack, Text } from "metabase/ui";import type {+ ConcreteTableId,DatabaseId,PythonTransformTableAliases,Table,@@ -174,6 +176,13 @@ function SelectionInput({}) {const table = availableTables.find((table) => table.id === selection.tableId);+ const selectedTableIds = useMemo((): ConcreteTableId[] => {+ return Object.values(tables).flatMap((v) => {+ const id = extractTableId(v);+ return id != null ? [id] : [];+ });+ }, [tables]);+function handleAliasChange(newAlias: string) {const newSelection = {...selection,@@ -218,7 +227,7 @@ function SelectionInput({<TableSelectordatabase={database}table={table}- selectedTableIds={Object.values(tables)}+ selectedTableIds={selectedTableIds}onChange={handleTableChange}onRemove={onRemove}availableTables={availableTables}diff --git a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.tsindex 69666c2fc77..793ac59e6d2 100644--- a/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts+++ b/enterprise/frontend/src/metabase-enterprise/transforms-python/components/PythonTransformEditor/PythonDataPicker/utils.ts@@ -1,4 +1,5 @@import { slugify as toSlug } from "metabase/lib/formatting";+import { extractTableId } from "metabase/transforms/utils";import type { PythonTransformTableAliases } from "metabase-types/api";import type { TableSelection } from "./types";@@ -7,8 +8,8 @@ export function getInitialTableSelections(tables: PythonTransformTableAliases | undefined,) {if (tables && Object.keys(tables).length > 0) {- return Object.entries(tables).map(([alias, tableId]) => ({- tableId,+ return Object.entries(tables).map(([alias, value]) => ({+ tableId: extractTableId(value),alias,}));}diff --git a/frontend/src/metabase/transforms/utils.ts b/frontend/src/metabase/transforms/utils.tsindex bd2e61ea97f..05a2e5592cf 100644--- a/frontend/src/metabase/transforms/utils.ts+++ b/frontend/src/metabase/transforms/utils.ts@@ -8,9 +8,11 @@ import * as Lib from "metabase-lib";import type Metadata from "metabase-lib/v1/metadata/Metadata";import type {CollectionNamespace,+ ConcreteTableId,Database,DatabaseId,DraftTransformSource,+ PythonTransformTableAliases,TemplateTag,Transform,TransformRun,@@ -135,11 +137,78 @@ export function isSameSource(return Lib.areLegacyQueriesEqual(source1.query, source2.query);}if (source1.type === "python" && source2.type === "python") {- return _.isEqual(source1, source2);+ return _.isEqual(+ {+ ...source1,+ "source-tables": collapseSourceTableReferences(+ source1["source-tables"],+ ),+ },+ {+ ...source2,+ "source-tables": collapseSourceTableReferences(+ source2["source-tables"],+ ),+ },+ );}return false;}+/**+ * Extract a ConcreteTableId from a source-tables value, which may be either+ * a bare integer or a map with table_id (from backend normalization).+ */+export function extractTableId(+ value: ConcreteTableId | Record<string, unknown>,+): ConcreteTableId | undefined {+ if (typeof value === "number") {+ return value;+ }+ if (typeof value === "object" && value != null) {+ const id = (value as Record<string, unknown>)["table_id"];+ if (typeof id === "number") {+ return id as ConcreteTableId;+ }+ }+ return undefined;+}++const collapseCache = new WeakMap<+ Record<string, unknown>,+ PythonTransformTableAliases+>();++/**+ * Collapse source-table references to plain { alias: tableId } form.+ * The backend may return map refs (with table_id, schema, etc.) instead of+ * bare integer IDs. Returns the original map unchanged if all values are+ * already plain integers. Results are memoized by object reference.+ */+export function collapseSourceTableReferences(+ tables: Record<string, unknown>,+): PythonTransformTableAliases {+ const cached = collapseCache.get(tables);+ if (cached) {+ return cached;+ }+ const values = Object.values(tables);+ if (values.every((v) => typeof v === "number")) {+ return tables as PythonTransformTableAliases;+ }+ const result: PythonTransformTableAliases = {};+ for (const [key, value] of Object.entries(tables)) {+ const id = extractTableId(+ value as ConcreteTableId | Record<string, unknown>,+ );+ if (id != null) {+ result[key] = id;+ }+ }+ collapseCache.set(tables, result);+ return result;+}+export function isSourceEmpty(source: DraftTransformSource,databaseId: DatabaseId,diff --git a/resources/python-sources/.lock b/resources/python-sources/.lockdeleted file mode 100644index e69de29bb2d..00000000000diff --git a/src/metabase/collections_rest/api.clj b/src/metabase/collections_rest/api.cljindex 033ae6114bf..f0ee92bc1bd 100644--- a/src/metabase/collections_rest/api.clj+++ b/src/metabase/collections_rest/api.clj@@ -28,7 +28,7 @@[metabase.request.core :as request][metabase.revisions.core :as revisions][metabase.transforms.feature-gating :as transforms.gating]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms.util :as transforms.u][metabase.upload.core :as upload][metabase.util :as u][metabase.util.honey-sql-2 :as h2x]@@ -488,7 +488,7 @@(defmethod collection-children-query :transform[_model collection {:keys [pinned-state]}]- (let [enabled-types (transforms.util/enabled-source-types-for-user)]+ (let [enabled-types (transforms.u/enabled-source-types-for-user)]{:select [:id :collection_id :name [(h2x/literal "transform") :model] :description :entity_id]:from [[:transform :transform]]:where [:anddiff --git a/src/metabase/core/init.clj b/src/metabase/core/init.cljindex 3fc57750572..3e546ff3f53 100644--- a/src/metabase/core/init.clj+++ b/src/metabase/core/init.clj@@ -65,6 +65,7 @@[metabase.task-history.init][metabase.testing-api.init][metabase.tiles.init]+ [metabase.transforms-base.init][metabase.transforms.init][metabase.types.init][metabase.upload.init]diff --git a/src/metabase/models/transforms/transform.clj b/src/metabase/models/transforms/transform.cljindex 531470506eb..53ec780df1d 100644--- a/src/metabase/models/transforms/transform.clj+++ b/src/metabase/models/transforms/transform.clj@@ -15,8 +15,9 @@[metabase.search.core :as search.core][metabase.search.ingestion :as search][metabase.search.spec :as search.spec]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.log :as log][methodical.core :as methodical]@@ -33,8 +34,8 @@(defmethod mi/can-read? :model/Transform([instance](and (api/is-data-analyst?)- (transforms.util/source-tables-readable? instance)- (transforms.util/check-feature-enabled instance)))+ (transforms.u/source-tables-readable? instance)+ (transforms.u/check-feature-enabled instance)))([_model pk](when-let [transform (t2/select-one :model/Transform :id pk)](mi/can-read? transform))))@@ -60,10 +61,10 @@;; Inline can-write? logic since instance is a plain map without model metadata.;; can-write? requires: can-read?, has-db-transforms-permission?, and transforms-editable?;; can-read? requires: is-superuser? OR (is-data-analyst? AND source-tables-readable?)- (let [source-db-id (or (:source_database_id instance) (transforms.i/source-db-id instance))]+ (let [source-db-id (or (:source_database_id instance) (transforms-base.i/source-db-id instance))](and (or api/*is-superuser?*(and api/*is-data-analyst?*- (transforms.util/source-tables-readable? instance)))+ (transforms.u/source-tables-readable? instance)))(perms/has-db-transforms-permission? api/*current-user-id* source-db-id)(remote-sync/transforms-editable?))))@@ -72,7 +73,10 @@[source-tables](update-vals source-tables #(if (map? %) (update-keys % keyword) %)))-(defn- transform-source-out [m]+(defn transform-source-out+ "Deserialize a transform source map from JSON storage format.+ Normalizes queries, keywordizes types and source-table refs."+ [m](-> mmi/json-out-without-keywordization(update-keys keyword)@@ -81,9 +85,12 @@(m/update-existing :type keyword)(m/update-existing :source-incremental-strategy #(update-keys % keyword))))-(defn- transform-source-in [m]+(defn transform-source-in+ "Serialize a transform source map for JSON storage.+ Normalizes source-tables and prepares queries for serialization."+ [m](-> m- (m/update-existing :source-tables transforms.util/normalize-source-tables)+ (m/update-existing :source-tables transforms-base.u/normalize-source-tables)(m/update-existing :query (comp lib/prepare-for-serialization lib-be/normalize-query))mi/json-in))@@ -102,7 +109,7 @@(collection/check-collection-namespace :model/Transform collection_id)(when collection_id(collection/check-allowed-content :model/Transform collection_id))- (let [target-db-id (transforms.i/target-db-id transform)+ (let [target-db-id (transforms-base.i/target-db-id transform);; This is defensive code to cope with some tests for remote sync, where we deserialize a transform;; with a concrete database id within it, for a potentially non-existent database.;; In practice, our serialized representation should not contain any database ids, and this should@@ -114,9 +121,9 @@(-> transform(assoc-in [:target :database] target-db-id)(assoc- :source_type (transforms.util/transform-source-type source)+ :source_type (transforms-base.u/transform-source-type source):target_db_id (when valid-db-id? target-db-id)- :source_database_id (or source_database_id (transforms.i/source-db-id transform))))))+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform))))))(t2/define-before-update :model/Transform[{:keys [source source_database_id] :as transform}]@@ -125,17 +132,17 @@(collection/check-allowed-content :model/Transform new-collection))(cond-> transformsource- (assoc :source_type (transforms.util/transform-source-type source)- :source_database_id (or source_database_id (transforms.i/source-db-id transform)))+ (assoc :source_type (transforms-base.u/transform-source-type source)+ :source_database_id (or source_database_id (transforms-base.i/source-db-id transform)))(or (:source (t2/changes transform)) (:target (t2/changes transform)));; No database existence check added here, unlike for insert. Just allow updates for an invalid target to fail.- (assoc :target_db_id (transforms.i/target-db-id transform))))+ (assoc :target_db_id (transforms-base.i/target-db-id transform))))(t2/define-after-select :model/Transform[{:keys [source] :as transform}](if source- (assoc transform :source_type (transforms.util/transform-source-type source))+ (assoc transform :source_type (transforms-base.u/transform-source-type source))transform))(methodical/defmethod t2/batched-hydrate [:model/TransformRun :transform]@@ -295,7 +302,7 @@"Fetch tables with their fields. The tables show up under the `:table` property."[transforms](let [table-key-fn (fn [{:keys [target] :as transform}]- [(transforms.i/target-db-id transform) (:schema target) (:name target)])+ [(transforms-base.i/target-db-id transform) (:schema target) (:name target)])table-keys (into #{} (map table-key-fn) transforms)table-keys-with-schema (filter second table-keys)table-keys-without-schema (keep (fn [[db-id schema table-name]]diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex ce6ca9432be..9dce8efb814 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -7,11 +7,12 @@[metabase.models.transforms.transform-run][metabase.models.transforms.transform-run-cancelation][metabase.models.transforms.transform-tag]+ [metabase.transforms-base.ordering]+ [metabase.transforms-base.util][metabase.transforms.canceling][metabase.transforms.crud][metabase.transforms.execute][metabase.transforms.jobs]- [metabase.transforms.ordering][metabase.transforms.schedule][metabase.transforms.settings][metabase.transforms.util]@@ -21,14 +22,15 @@(p/import-vars[metabase.transforms.settingstransform-timeout]- [metabase.transforms.util- add-source-readable+ [metabase.transforms-base.utilnative-query-transform?python-transform?query-transform?transform-source-databasetransform-source-type- transform-type+ transform-type]+ [metabase.transforms.util+ add-source-readableis-temp-transform-table?][metabase.transforms.crudpython-source-table-ref->table-id@@ -47,7 +49,7 @@cancel-run!][metabase.transforms.executeexecute!]- [metabase.transforms.ordering+ [metabase.transforms-base.orderingtransform-orderingget-transform-cycle][metabase.transforms.jobsdiff --git a/src/metabase/transforms/crud.clj b/src/metabase/transforms/crud.cljindex 4da9e2ef79d..59f1438012d 100644--- a/src/metabase/transforms/crud.clj+++ b/src/metabase/transforms/crud.clj@@ -13,9 +13,10 @@[metabase.models.interface :as mi][metabase.models.transforms.transform :as transform.model][metabase.query-processor.compile :as qp.compile]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.ordering :as transforms.ordering]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.ordering :as transforms-base.ordering]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.i18n :refer [deferred-tru]][metabase.util.log :as log]@@ -31,7 +32,7 @@We now supported table-ref as source but since FE is still expecting table-id we need to temporarily do this.Should update FE to fully use table-ref"[transform]- (if (transforms.util/python-transform? transform)+ (if (transforms-base.u/python-transform? transform)(update-in transform [:source :source-tables](fn [source-tables](update-vals source-tables #(if (int? %) % (:table_id %)))))@@ -40,9 +41,9 @@(defn check-database-feature"Check that the target database supports the required features for this transform."[transform]- (let [database (api/check-400 (t2/select-one :model/Database (transforms.i/target-db-id transform))+ (let [database (api/check-400 (t2/select-one :model/Database (transforms-base.i/target-db-id transform))(deferred-tru "The target database cannot be found."))- features (transforms.util/required-database-features transform)]+ features (transforms-base.u/required-database-features transform)](api/check-400 (not (:is_sample database))(deferred-tru "Cannot run transforms on the sample database."))(api/check-400 (not (:is_audit database))@@ -55,13 +56,13 @@(defn check-feature-enabled!"Check that the premium features required for this transform type are enabled."[transform]- (api/check (transforms.util/check-feature-enabled transform)+ (api/check (transforms.u/check-feature-enabled transform)[402 (deferred-tru "Premium features required for this transform type are not enabled.")]))(defn validate-transform-query!"Validate that the transform query is valid. Throws a 400 error if validation fails."[transform]- (when-let [error (transforms.util/validate-transform-query transform)]+ (when-let [error (transforms-base.u/validate-transform-query transform)](throw (ex-info (:error error)(assoc error:status-code 400)))))@@ -107,7 +108,7 @@be supported by all drivers or for all query types."[driver database-id query](some->> (extract-all-columns-from-query driver database-id query)- (filter (comp transforms.util/supported-incremental-filter-type? :base_type))+ (filter (comp transforms-base.u/supported-incremental-filter-type? :base_type))(mapv :name)))(defn validate-incremental-column-type!@@ -130,7 +131,7 @@checkpoint-filter-unique-key(let [column (lib/column-with-unique-key query checkpoint-filter-unique-key)](api/check-400 column (deferred-tru "Checkpoint column not found in query."))- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base-type column))+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base-type column))(deferred-tru "Checkpoint column type {0} is not supported. Only numeric and temporal types are supported for incremental filtering."(pr-str (:base-type column)))))@@ -138,7 +139,7 @@checkpoint-filter(when-some [column-metadata (seq (extract-all-columns-from-query driver-name database-id query))](when-some [column (first (filter #(= checkpoint-filter (:name %)) column-metadata))]- (api/check-400 (transforms.util/supported-incremental-filter-type? (:base_type column))+ (api/check-400 (transforms-base.u/supported-incremental-filter-type? (:base_type column))(deferred-tru "Checkpoint column ''{0}'' has unsupported type {1}. Only numeric and temporal columns are supported for incremental filtering."checkpoint-filter(pr-str (:base_type column)))))))))))@@ -146,30 +147,30 @@(defn get-transforms"Get a list of transforms."[& {:keys [last-run-start-time last-run-statuses tag-ids]}]- (let [enabled-types (transforms.util/enabled-source-types-for-user)]+ (let [enabled-types (transforms.u/enabled-source-types-for-user)](api/check-403 (seq enabled-types))(let [transforms (t2/select :model/Transform {:where [:in :source_type enabled-types]:order-by [[:id :asc]]})](->> (t2/hydrate transforms :last_run :transform_tag_ids :creator :owner)(into []- (comp (transforms.util/->date-field-filter-xf [:last_run :start_time] last-run-start-time)- (transforms.util/->status-filter-xf [:last_run :status] last-run-statuses)- (transforms.util/->tag-filter-xf [:tag_ids] tag-ids)- (map #(update % :last_run transforms.util/localize-run-timestamps))+ (comp (transforms-base.u/->date-field-filter-xf [:last_run :start_time] last-run-start-time)+ (transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)+ (transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)+ (map #(update % :last_run transforms-base.u/localize-run-timestamps))(map python-source-table-ref->table-id)))- transforms.util/add-source-readable))))+ transforms.u/add-source-readable))))(defn get-transform"Get a specific transform."[id](let [{:keys [target] :as transform} (api/read-check :model/Transform id)- target-table (transforms.util/target-table (transforms.i/target-db-id transform) target :active true)]+ target-table (transforms-base.u/target-table (transforms-base.i/target-db-id transform) target :active true)](-> transform(t2/hydrate :last_run :transform_tag_ids :creator :owner)- (u/update-some :last_run transforms.util/localize-run-timestamps)+ (u/update-some :last_run transforms-base.u/localize-run-timestamps)(assoc :table target-table)python-source-table-ref->table-id- transforms.util/add-source-readable)))+ transforms.u/add-source-readable)))(defn create-transform!"Create new transform in the appdb.@@ -177,7 +178,7 @@([body](create-transform! body nil))([body creator-id]- (when (transforms.util/query-transform? body)+ (when (transforms-base.u/query-transform? body)(validate-transform-query! body))(let [creator-id (or creator-id api/*current-user-id*)transform (t2/with-transaction [_]@@ -214,13 +215,13 @@(check-feature-enabled! new)(check-database-feature new)(validate-incremental-column-type! new)- (when (transforms.util/query-transform? old)+ (when (transforms-base.u/query-transform? old)(validate-transform-query! new)- (when-let [{:keys [cycle-str]} (transforms.ordering/get-transform-cycle new)]+ (when-let [{:keys [cycle-str]} (transforms-base.ordering/get-transform-cycle new)](throw (ex-info (str "Cyclic transform definitions detected: " cycle-str){:status-code 400}))))(api/check (not (and (not= (target-fields old) (target-fields new))- (transforms.util/target-table-exists? new)))+ (transforms-base.u/target-table-exists? new)))403(deferred-tru "A table with that name already exists.")))(t2/update! :model/Transform id (dissoc body :tag_ids))@@ -231,7 +232,7 @@(events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})(-> transformpython-source-table-ref->table-id- transforms.util/add-source-readable)))+ transforms.u/add-source-readable)))(defn delete-transform!"Delete a transform and publish the delete event."diff --git a/src/metabase/transforms/interface.clj b/src/metabase/transforms/interface.cljindex b6fb789d31a..f788462e90e 100644--- a/src/metabase/transforms/interface.clj+++ b/src/metabase/transforms/interface.clj@@ -1,32 +1,21 @@-(ns metabase.transforms.interface)+(ns metabase.transforms.interface+ "Interface for scheduled transform execution.-(defn- transform->transform-type- [transform]- (-> transform :source :type keyword))+ For core interface methods (source-db-id, target-db-id, table-dependencies, execute-base!),+ use metabase.transforms-base.interface directly.-(defmulti source-db-id- "Return the ID of the source database for a given `transform`. The source database is where the data originates from- before being transformed and written to the target destination.-- NOTE: Currently returns a single database ID, but we may want to change this to return multiple database IDs in the- future since transforms can have multiple input sources."- {:added "0.57.0" :arglists '([transform])}- transform->transform-type)--(defmulti target-db-id- "Return the ID of the target database for a given `transform`. The target database is the destination where the- transformed data will be written. Often this is the same database as the source database."- {:added "0.57.0" :arglists '([transform])}- transform->transform-type)+ This namespace defines the scheduled-execution-specific `execute!` multimethod+ which handles transform_run tracking."+ (:require+ [metabase.transforms-base.interface :as transforms-base.i]))+;; Scheduled execution multimethod - dispatches on transform type+;; This is separate from transforms-base.i/execute-base! which does NOT track transform_run(defmulti execute!- "Execute a transform operation. Runs the actual transformation process, which might involve running SQL queries,- Python scripts, or other transformation logic depending on the transform type.-- This method blocks and may take significant time depending on the data volume. Implementations- should handle errors gracefully and provide appropriate logging.+ "Execute a transform operation with transform_run tracking.- Returns nil (or a result that can be discarded).+ This method creates transform_run rows, tracks status, and handles cancellation.+ For base execution without database tracking, use [[metabase.transforms-base.interface/execute-base!]] instead.Options:- `:start-promise`@@ -45,24 +34,4 @@Do not use this directly. Use [[metabase.transforms.execute/execute!]] instead."{:added "0.57.0" :arglists '([transform options])}(fn [transform _options]- (transform->transform-type transform)))--(defmulti table-dependencies- "Return a set of logical table dependencies of the transform, including indirect dependencies via cards.- The transform execution system uses these dependencies to determine the correct order of execution- and to detect circular dependencies.-- Each dependency is represented as one of:- - `{:table <table-id>}`- A dependency on a table that exists and has been synced.- - `{:transform <transform-id>}`- A dependency on a table that does not yet exist, but is known to be the target of another transform.- Represents a 'placeholder' table (as we no table id / metadata) for the same purposes.- - `{:table-ref {:database_id <db-id> :schema <schema> :table <name>}}`- A dependency on a table by name, for cases where the table_id may not exist yet (e.g., when- bulk importing transforms that depend on each other's outputs). The ordering system will- resolve these to transforms by matching against target table definitions.-- An empty set indicates no dependencies."- {:added "0.57.0" :arglists '([transform])}- transform->transform-type)+ (transforms-base.i/transform->transform-type transform)))diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex b00121f4198..d733db93600 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -11,11 +11,11 @@[metabase.models.transforms.transform-run :as transform-run][metabase.revisions.core :as revisions][metabase.task.core :as task]+ [metabase.transforms-base.ordering :as transforms-base.ordering][metabase.transforms.execute :as transforms.execute][metabase.transforms.instrumentation :as transforms.instrumentation]- [metabase.transforms.ordering :as transforms.ordering][metabase.transforms.settings :as transforms.settings]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms.util :as transforms.u][metabase.util :as u][metabase.util.i18n :as i18n][metabase.util.log :as log]@@ -34,7 +34,7 @@found)))(defn- next-transform [ordering transforms-by-id complete]- (-> (transforms.ordering/available-transforms ordering #{} complete)+ (-> (transforms-base.ordering/available-transforms ordering #{} complete)firsttransforms-by-id))@@ -52,7 +52,7 @@(defn- get-plan [transform-ids](let [all-transforms (t2/select :model/Transform)- global-ordering (transforms.ordering/transform-ordering all-transforms)+ global-ordering (transforms-base.ordering/transform-ordering all-transforms)relevant-ids (get-deps global-ordering transform-ids)transforms-by-id (into {}(keep (fn [{:keys [id] :as transform}]@@ -60,7 +60,7 @@[id transform])))all-transforms)ordering (sorted-ordering (select-keys global-ordering relevant-ids) transforms-by-id)]- (when-let [cycle (transforms.ordering/find-cycle ordering)]+ (when-let [cycle (transforms-base.ordering/find-cycle ordering)](let [id->name (into {} (map (juxt :id :name)) all-transforms)](throw (ex-info (str "Cyclic transform definitions detected: "(str/join " → " (map id->name cycle)))@@ -78,7 +78,7 @@(Thread/sleep 2000))))(defn- run-transform! [run-id run-method user-id {transform-id :id :as transform}]- (if-not (transforms.util/check-feature-enabled transform)+ (if-not (transforms.u/check-feature-enabled transform)(log/warnf "Skip running transform %d due to lacking premium features" transform-id)(do(block-until-not-already-running transform-id)diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.cljindex e53845b5814..ee8c2751454 100644--- a/src/metabase/transforms/query_impl.clj+++ b/src/metabase/transforms/query_impl.clj@@ -1,99 +1,57 @@(ns metabase.transforms.query-impl(:require- [metabase.database-routing.core :as database-routing]+ [clojure.core.async :as a][metabase.driver :as driver][metabase.driver.connection :as driver.conn]- [metabase.driver.util :as driver.u]- [metabase.lib.schema.common :as schema.common]- [metabase.query-processor.compile :as qp.compile]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.instrumentation :as transforms.instrumentation][metabase.transforms.interface :as transforms.i]- [metabase.transforms.util :as transforms.util]- [metabase.util.i18n :as i18n]+ [metabase.transforms.util :as transforms.u][metabase.util.log :as log]- [metabase.util.malli.registry :as mr][toucan2.core :as t2]))(set! *warn-on-reflection* true)-(defmethod transforms.i/source-db-id :query- [transform]- (-> transform :source :query :database))--(defmethod transforms.i/target-db-id :query- [transform]- ;; For query transforms, the target needs to match the source, so use the query as the source of truth.- (or (-> transform :source :query :database)- ;; Fallback to using a configured value.- (get-in transform [:target :database])- (:target_db_id transform)))--(mr/def ::transform-details- [:map- [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]- [:conn-spec :any]- [:query ::qp.compile/compiled]- [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])--(mr/def ::transform-opts- [:map- [:overwrite? :boolean]])--(defn- transform-opts [_transform-details]- ;; once we have more than just :table and :table-incremental as transform-types,- ;; then we can dispatch on :target-incremental-strategy- {})--(defn- throw-if-db-routing-enabled [transform driver database]- (when (database-routing/db-routing-enabled? database)- (throw (ex-info (i18n/tru "Failed to run the transform ({0}) because the database ({1}) has database routing turned on. Running transforms on databases with db routing enabled is not supported."- (:name transform)- (:name database))- {:driver driver, :database database}))))-(defn- run-mbql-transform!([transform] (run-mbql-transform! transform nil))- ([{:keys [id source target owner_user_id creator_id] :as transform} {:keys [run-method start-promise user-id]}]+ ([{:keys [id source target owner_user_id creator_id] :as transform}+ {:keys [run-method start-promise user-id]}](try- (let [db (get-in source [:query :database])- {driver :engine :as database} (t2/select-one :model/Database db)- ;; important to test routing before calling compile-source (whose qp middleware will also throw)- _ (throw-if-db-routing-enabled transform driver database)]+ (let [db (t2/select-one :model/Database (get-in source [:query :database]))+ driver (:engine db)+ _ (transforms-base.u/throw-if-db-routing-enabled! transform db)+ run-user-id (if (and (= run-method :manual) user-id)+ user-id+ (or owner_user_id creator_id))+ {run-id :id} (transforms.u/try-start-unless-already-running id run-method run-user-id)]+ (when start-promise (deliver start-promise [:started run-id]))(driver.conn/with-write-connection- (let [conn-spec (driver/connection-spec driver database)- transform-details {:db-id db- :database database- :transform-id id- :transform-type (keyword (:type target))- :conn-spec conn-spec- :query (transforms.util/compile-source transform)- :output-schema (:schema target)- :output-table (transforms.util/qualified-table-name driver target)}- opts (transform-opts transform-details)- features (transforms.util/required-database-features transform)- ;; For manual runs, use the triggering user; for cron, use owner/creator- run-user-id (if (and (= run-method :manual) user-id)- user-id- (or owner_user_id creator_id))]- (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)- (throw (ex-info "The database does not support the requested transform target type."- {:driver driver, :database database, :features features})))- ;; mark the execution as started and notify any observers- (let [{run-id :id} (transforms.util/try-start-unless-already-running id run-method run-user-id)]- (when start-promise- (deliver start-promise [:started run-id]))- (log/info "Executing transform" id "with target" (pr-str target)- (when (driver.conn/write-connection-requested?)- " using write connection"))- (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]- (transforms.util/run-cancelable-transform!- run-id driver transform-details- (fn [_cancel-chan]- (driver/run-transform! driver transform-details opts))))- (transforms.util/handle-transform-complete!- :run-id run-id- :transform transform- :db database)))))+ (log/info "Executing transform" id "with target" (pr-str target)+ (when (driver.conn/write-connection-requested?) " using write connection"))+ (let [conn-spec (driver/connection-spec driver db)+ transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}]+ (transforms.instrumentation/with-stage-timing [run-id [:computation :mbql-query]]+ (transforms.u/run-cancelable-transform!+ run-id driver transform-details+ (fn [cancel-chan]+ (let [result (transforms-base.i/execute-base!+ transform+ {:cancelled? #(boolean (a/poll! cancel-chan))+ :run-id run-id+ :with-stage-timing-fn (fn [rid stage thunk]+ (transforms.instrumentation/with-stage-timing [rid stage]+ (thunk)))})]+ ;; Bridge result-map to exception-based flow for run-cancelable-transform!+ (when-not (= :succeeded (:status result))+ (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))+ result))))+ ;; Post-processing: sync, transform_id, events, secondary indexes (after succeed-started-run!)+ (transforms-base.u/complete-execution! transform+ {:run-id run-id+ :with-stage-timing-fn (fn [rid stage thunk]+ (transforms.instrumentation/with-stage-timing [rid stage]+ (thunk)))}))))(catch Throwable t(if (= :already-running (:error (ex-data t)))(log/warnf "Transform %d is already running" id)diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex f0c61c0d308..bdc495e2801 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -1,116 +1,38 @@(ns metabase.transforms.util+ "Transform utilities for scheduled execution.++ Most pure utilities are in metabase.transforms-base.util and re-exported here.+ This namespace adds scheduled-execution-specific functions that depend on+ transform_run tracking, cancellation, instrumentation, and feature gating."(:require- [buddy.core.codecs :as codecs]- [buddy.core.hash :as buddy-hash][clojure.core.async :as a][clojure.string :as str]- [java-time.api :as t][metabase.api.common :as api][metabase.driver :as driver][metabase.driver.sql-jdbc :as sql-jdbc]- [metabase.driver.sql.query-processor :as sql.qp][metabase.driver.util :as driver.u]- [metabase.events.core :as events]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.common :as lib.schema.common][metabase.models.interface :as mi][metabase.models.transforms.transform-run :as transform-run]- [metabase.query-processor :as qp]- [metabase.query-processor.compile :as qp.compile]- [metabase.query-processor.middleware.add-remaps :as remap]- [metabase.query-processor.middleware.catch-exceptions :as qp.catch-exceptions]- [metabase.query-processor.parameters.dates :as params.dates][metabase.query-processor.pipeline :as qp.pipeline]- [metabase.query-processor.preprocess :as qp.preprocess]- [metabase.sync.core :as sync]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.canceling :as canceling][metabase.transforms.feature-gating :as transforms.gating]- [metabase.transforms.instrumentation :as transforms.instrumentation]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.schema :as transforms.schema][metabase.transforms.settings :as transforms.settings][metabase.util :as u]- [metabase.util.date-2 :as u.date]- [metabase.util.honey-sql-2 :as h2x]- [metabase.util.log :as log]- [metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr][toucan2.core :as t2])(:import- (java.sql SQLException)- (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)- (java.util Date)))+ (java.sql SQLException)))(set! *warn-on-reflection* true)-(defn qualified-table-name- "Return the name of the target table of a transform as a possibly qualified symbol."- [_driver {:keys [schema name]}]- (if schema- (keyword schema name)- (keyword name)))--(defn transform-type- "Get the type of a transform"- [transform]- (-> transform :source :type keyword))--(defn query-transform?- "Check if this is a query transform: native query / mbql query."- [transform]- (= :query (transform-type transform)))--(defn native-query-transform?- "Check if this is a native query transform.- Note: The transform should be normalized (via `normalize-transform`) before calling this function."- [transform]- (when (query-transform? transform)- (let [query (-> transform :source :query)]- (lib/native-only-query? query))))--(defn python-transform?- "Check if this is a Python transform."- [transform]- (= :python (transform-type transform)))--(defn transform-source-database- "Get the source database from a transform"- [transform]- (case (transform-type transform)- :query (-> transform :source :query :database)- :python (-> transform :source :source-database)))--(defn normalize-transform- "Normalize a transform's source query, similar to how transforms are normalized when read from the database.- This should be called on transforms before processing them to ensure queries are in the expected format."- [transform]- (if (and (map? transform)- (= (:type transform) "transform")- (get-in transform [:source :query]))- (update-in transform [:source :query] lib-be/normalize-query)- transform))--(defn transform-source-type- "Returns the type of a transform's source: :python, :native, or :mbql.- Throws if the source type cannot be detected.- Note: The transform should be normalized (via `normalize-transform`) before calling this function."- [source]- (case (keyword (:type source))- :python :python- :query (if (lib/native-only-query? (:query source))- :native- :mbql)- (throw (ex-info (str "Unknown transform source type: " (:type source))- {:source source}))))+;;; ------------------------------------------------- Feature Gating -------------------------------------------------(defn check-feature-enabled"Checking whether we have proper feature flags for using a given transform."[transform](cond- (query-transform? transform) (transforms.gating/query-transforms-enabled?)- (python-transform? transform) (transforms.gating/python-transforms-enabled?)+ (transforms-base.u/query-transform? transform) (transforms.gating/query-transforms-enabled?)+ (transforms-base.u/python-transform? transform) (transforms.gating/python-transforms-enabled?):else false))(defn enabled-source-types-for-user@@ -161,6 +83,8 @@transform-or-transforms)(assoc transform-or-transforms :source_readable (source-tables-readable? transform-or-transforms))))+;;; ------------------------------------------------- Scheduled Execution -------------------------------------------------+(defn- duplicate-key-violation?"Check if an exception is a duplicate key violation.Returns true for Postgres, MySQL/MariaDB, and H2 duplicate key errors."@@ -204,554 +128,16 @@(transform-run/succeed-started-run! run-id)ret)(catch Throwable t- (transform-run/fail-started-run! run-id {:message (ex-message-fn t)})+ (if (:timeout (ex-data t))+ (transform-run/timeout-run! run-id {:message (ex-message-fn t)})+ (transform-run/fail-started-run! run-id {:message (ex-message-fn t)}))(throw t))(finally(canceling/chan-end-run! run-id))))-(declare activate-table-and-mark-computed! target-table)--(defn sync-target!- "Sync target of a transform"- [target database]- ;; sync the new table (note that even a failed sync status means that the execution succeeded)- (log/info "Syncing target" (pr-str target) "for transform")- (activate-table-and-mark-computed! database target))--(defn target-table-exists?- "Test if the target table of a transform already exists."- [{:keys [target] :as transform}]- (let [db-id (transforms.i/target-db-id transform)- {driver :engine :as database} (t2/select-one :model/Database db-id)]- (driver/table-exists? driver database target)))--(defn target-table- "Load the `target` table of a transform from the database specified by `database-id`."- [database-id target & kv-args]- (some-> (apply t2/select-one :model/Table- :db_id database-id- :schema (:schema target)- :name (:name target)- kv-args)- (t2/hydrate :db)))--(defn- sync-table!- ([database target] (sync-table! database target nil))- ([database target {:keys [create?]}]- (when-let [table (or (target-table (:id database) target)- (when create?- (sync/create-table! database (select-keys target [:schema :name :data_source :data_authority]))))]- (sync/sync-table! table)- table)))--(defn activate-table-and-mark-computed!- "Activate table for `target` in `database` in the app db."- [database target]- (when-let [table (sync-table! database (assoc target- :data_authority :computed- :data_source :metabase-transform)- {:create? true})]- (when-not (:active table)- (t2/update! :model/Table (:id table) {:active true}))- table))--(defn deactivate-table!- "Deactivate table for `target` in `database` in the app db."- [database target]- (when-let [table (sync-table! database target)]- ;; TODO this should probably be a function in the sync module- (t2/update! :model/Table (:id table) {:active false})))--(defn delete-target-table!- "Delete the target table of a transform and sync it from the app db."- [{:keys [id target], :as transform}]- (when target- (let [target (update target :type keyword)- database-id (transforms.i/target-db-id transform)]- (when database-id- (if-let [{driver :engine :as database} (t2/select-one :model/Database database-id)]- (do- (driver/drop-transform-target! driver database target)- (log/info "Deactivating target " (pr-str target) "for transform" id)- (deactivate-table! database target))- (log/warnf "Skipping drop of transform target %s for transform %d: database %d not found"- (pr-str target) id database-id))))))--(defn delete-target-table-by-id!- "Delete the target table of the transform specified by `transform-id`."- [transform-id]- (delete-target-table! (t2/select-one :model/Transform transform-id)))--(defn massage-sql-query- "Adjusts mbql query for use in a transform."- [query]- (-> query- remap/disable-remaps- lib/disable-default-limit))--(defn- checkpoint-incremental?- "Returns true if `source` uses checkpoint-based incremental strategy."- [source]- (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))--(defn supported-incremental-filter-type?- "Returns true if the given base-type is supported for incremental filtering.-- We only support temporal (timestamp/tz) and numeric (int/float) types."- [base-type]- (or (isa? base-type :type/Temporal)- (isa? base-type :type/Number)))--(defn- source->checkpoint-filter-unique-key- "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."- [query source-incremental-strategy]- (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))--(defn- source->checkpoint-filter-column- "Resolve the checkpoint filter column for an incremental transform.-- Tries to resolve the column using the unique key first.- Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.-- Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).- Throws an exception if the column type is not supported."- [query source-incremental-strategy table metadata-provider]- (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]- (when-some [{column-name :name- :keys [base-type]- :as column}- (cond- checkpoint-filter-unique-key- (source->checkpoint-filter-unique-key query source-incremental-strategy)- checkpoint-filter- (when-some [field-id (t2/select-one-pk :model/Field- :table_id (:id table)- :name checkpoint-filter)]- (lib.metadata/field metadata-provider field-id)))]- (when-not (supported-incremental-filter-type? base-type)- (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "- "Only numeric and temporal columns are supported for incremental filtering.")- {:column-name column-name- :base-type base-type})))- column)))--(defn next-checkpoint- "Build a query to compute the MAX of the checkpoint column from the target table.-- Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),- or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."- [{:keys [source target] :as transform}]- (let [db-id (transforms.i/target-db-id transform)]- (when (checkpoint-incremental? source)- (when-let [table (target-table db-id target)]- (let [metadata-provider (lib-be/application-database-metadata-provider db-id)- table-metadata (lib.metadata/table metadata-provider (:id table))- query (lib/query metadata-provider table-metadata)]- (when-let [filter-column (source->checkpoint-filter-column query- (:source-incremental-strategy source)- table metadata-provider)]- {:query (-> query (lib/aggregate (lib/max filter-column)))- :filter-column filter-column}))))))--(defn- next-checkpoint-value- "Execute the checkpoint query and normalize the result for database insertion.- Returns `nil` if the target table is empty."- [{:keys [query filter-column]}]- (let [{:keys [base-type]} filter-column- v (some-> query qp/process-query :data :rows first first)]- ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right- ;; shape for reinsertion- (cond- (nil? v)- nil-- (isa? base-type :type/Integer)- (bigint v)-- ;; any other number that's not an integer, should be a decimal/float- (number? v)- (bigdec v)-- :else v)))--(defn preprocess-incremental-query- "Add checkpoint checkpoint filtering to a query for incremental execution.-- For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.- For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.- Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."- [query source-incremental-strategy checkpoint]- (if-let [checkpoint-value (next-checkpoint-value checkpoint)]- (if (lib/native? query)- ;; native query with explicit checkpoint filter- (if (get-in query [:stages 0 :template-tags "checkpoint"])- (update query :parameters conj- {:type (if (number? checkpoint-value) :number :text)- :target [:variable [:template-tag "checkpoint"]]- :value checkpoint-value})- query)- ;; mbql query- (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))- query))--(defn- post-process-incremental-query- "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.-- Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "- [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]- (let [{:keys [checkpoint-filter]} source-incremental-strategy]- (if (and (lib/native? (:query source))- (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))- (next-checkpoint-value checkpoint))- (let [wrap-query (fn [query]- (let [honeysql-query {:select [:*]- :from [[[:raw (str "(" query ")")] :subquery]]- :where [:> (h2x/identifier :field checkpoint-filter)- [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]- (first (sql.qp/format-honeysql driver honeysql-query))))]- (update outer-query :query wrap-query))- outer-query)))--(mu/defn validate-transform-query :- [:maybe [:map [:error :string]]]- "Verifies that a query transform's query can actually be run as is. Returns nil on success and an error map on failure."- [{:keys [source]}]- (case (keyword (:type source))- :query- (try- (qp.preprocess/preprocess (:query source))- nil- (catch Exception e- (qp.catch-exceptions/exception-response e)))))--(defn compile-source- "Compile the source query of a transform to SQL, applying incremental filtering if required."- [{:keys [source] :as transform}]- (let [{:keys [source-incremental-strategy] query-type :type} source]- (case (keyword query-type)- :query- (let [checkpoint (next-checkpoint transform)- query (:query source)- driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]- (binding [driver/*compile-with-inline-parameters*- (or (= :clickhouse driver)- driver/*compile-with-inline-parameters*)]- (-> query- (preprocess-incremental-query source-incremental-strategy checkpoint)- massage-sql-query- qp.compile/compile- (post-process-incremental-query driver source checkpoint)))))))--(defn required-database-features- "Returns the database features necessary to execute `transform`."- [transform]- (if (python-transform? transform)- [:transforms/python]- [:transforms/table]))--(defn ->instant- "Convert a temporal value `t` to an Instant in the system timezone."- ^Instant [t]- (when t- (condp instance? t- Instant t- Date (.toInstant ^Date t)- OffsetDateTime (.toInstant ^OffsetDateTime t)- ZonedDateTime (.toInstant ^ZonedDateTime t)- LocalDateTime (recur (.atZone ^LocalDateTime t (t/zone-id)))- String (recur (u.date/parse t))- LocalTime (recur (.atDate ^LocalTime t (t/local-date)))- OffsetTime (recur (.atDate ^OffsetTime t (t/local-date)))- LocalDate (recur (.atStartOfDay ^LocalDate t))- (throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")- {:temporal t})))))--(defn utc-timestamp-string- "Convert the timestamp t to a string encoding the it in the system timezone."- [t]- (-> t ->instant str))--(defn localize-run-timestamps- "Convert the timestamps of a `run` to ISO strings in UTC."- [run]- (-> run- (u/update-some :start_time utc-timestamp-string)- (u/update-some :end_time utc-timestamp-string)))--(mr/def ::column-definition- [:map- [:name :string]- [:type ::lib.schema.common/base-type]- [:nullable? {:optional true} :boolean]])--(mr/def ::table-definition- [:map- [:name :keyword]- [:columns [:sequential ::column-definition]]- [:primary-key {:optional true} [:sequential :string]]])--(mu/defn create-table-from-schema!- "Create a table from a table-schema"- [driver :- :keyword- database-id :- pos-int?- table-schema :- ::table-definition]- (let [{:keys [columns] table-name :name} table-schema- column-definitions (mapv (fn [{:keys [name type database-type]}]- (let [db-type (if database-type- [[:raw database-type]]- (try- (driver/type->database-type driver type)- (catch IllegalArgumentException _- (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)- (driver/type->database-type driver :type/Text))))]- [name db-type]))- columns)- primary-key-opts (select-keys table-schema [:primary-key])]- (log/infof "Creating table %s with %d columns" table-name (count columns))- (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))--(defn drop-table!- "Drop a table in the database."- [driver database-id table-name]- (log/infof "Dropping table %s" table-name)- (driver/drop-table! driver database-id table-name))--(defn rename-tables!- "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.- This is a simpler, composable operation that only handles renaming."- [driver database-id rename-map]- (log/infof "Renaming tables: %s" (pr-str rename-map))- (driver/rename-tables! driver database-id rename-map))-(defn is-temp-transform-table?"Return true when `table` matches the transform temporary table naming pattern and transforms are enabled."[table](boolean(when-let [table-name (and (transforms.gating/any-transforms-enabled?) (:name table))](str/starts-with? (u/lower-case-en table-name) driver.u/transform-temp-table-prefix))))--;;; ------------------------------------------------- Source Table Resolution -------------------------------------------(def ^:private ^:const batch-lookup-chunk-size- "Maximum number of table refs to query at once to avoid SQL query size limits."- 100)--(defn batch-lookup-table-ids- "Batch lookup table IDs from ref maps. Returns {[db_id schema name] -> table_id}.- Queries the exact conjunction of each [database_id, schema, table] triple rather than- a Cartesian product of all values. Uses chunking to avoid query size limits."- [refs]- (when (seq refs)- (let [unique-refs (distinct (map (juxt :database_id :schema :table) refs))- ref->clause (fn [[db-id schema table-name]]- [:and- [:= :db_id db-id]- (if (some? schema)- [:= :schema schema]- [:is :schema nil])- [:= :name table-name]])- fetch-batch (fn [batch]- (t2/select-fn->fn (juxt :db_id :schema :name) :id- [:model/Table :id :db_id :schema :name]- {:where (into [:or] (map ref->clause batch))}))]- (into {} (mapcat fetch-batch) (partition-all batch-lookup-chunk-size unique-refs)))))--(defn- source-table-ref->key- "Convert a source table ref map to a lookup key [db_id schema name]."- [{:keys [database_id schema table]}]- [database_id schema table])--(defn- missing-table-id?- "Returns true if `v` is a source table ref map that needs table_id lookup."- [v]- (and (map? v) (nil? (:table_id v))))--(defn normalize-source-tables- "Normalize source-tables to consistent map format {:database_id :schema :table :table_id}.-- The old format stored just integer table IDs. New transforms store maps on write.- Old data is converted on read via transform-source-out for backwards compatibility.-- Throws if an integer table ID references a non-existent table.- Map refs with non-existent tables get nil table_id (resolved later at execute time)."- [source-tables]- (let [int-table-ids (into #{} (filter int?) (vals source-tables))- int-id->metadata (when (seq int-table-ids)- (t2/select-pk->fn (fn [{:keys [db_id schema name]}]- {:database_id db_id :schema schema :table name})- [:model/Table :id :db_id :schema :name]- :id [:in int-table-ids]))- missing-ids (when (seq int-table-ids)- (remove int-id->metadata int-table-ids))- refs-needing-id (filter missing-table-id? (vals source-tables))- ref-lookup (or (batch-lookup-table-ids refs-needing-id) {})]- (when (seq missing-ids)- (throw (ex-info (str "Tables not found for ids: " (str/join ", " (sort missing-ids)))- {:table_ids (vec missing-ids)})))- (update-vals source-tables- (fn [v]- (cond- (int? v) (assoc (int-id->metadata v) :table_id v)- (:table_id v) v- :else (assoc v :table_id (ref-lookup (source-table-ref->key v))))))))--(defn resolve-source-tables- "Resolve source-tables to {alias -> table_id}. Throws if any table not found.- For execute time - all entries must resolve to valid table IDs.- Handles both integer IDs (old format) and map refs (new format)."- [source-tables]- (let [needs-lookup (filter missing-table-id? (vals source-tables))- lookup (or (batch-lookup-table-ids needs-lookup) {})- resolved (u/for-map [[alias v] source-tables]- [alias (if (int? v)- v- (or (:table_id v) (lookup (source-table-ref->key v))))])- unresolved (for [[alias table-id] resolved- :when (nil? table-id)- :let [v (get source-tables alias)]]- {:alias alias- :table (if-let [schema (:schema v)]- (str schema "." (:table v))- (:table v))- :ref v})]- (when (seq unresolved)- (throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved)))- {:unresolved unresolved- :transform-message "Input table not found"})))- resolved))--(defn- matching-timestamp?- [job field-path {:keys [start end]}]- (when-let [field-instant (->instant (get-in job field-path))]- (let [parse #(-> % u.date/parse ->instant)]- ;; logic here is to find when it's not matching and invert this- (not (or (and start (.isBefore field-instant (parse start)))- (and end (.isAfter field-instant (parse end))))))))--(defn ->date-field-filter-xf- "Returns an xform for a date filter."- [field-path filter-value]- (if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]- (filter #(matching-timestamp? % field-path range))- identity))--(defn ->status-filter-xf- "Returns an xform for a transform run status filter."- [field-path statuses]- (if-let [statuses (->> statuses (map keyword) set not-empty)]- (filter #(statuses (get-in % field-path)))- identity))--(defn ->tag-filter-xf- "Returns an xform for a transform tag filter."- [field-path tag-ids]- (if-let [tag-ids (-> tag-ids set not-empty)]- (filter #(some tag-ids (get-in % field-path)))- identity))--(def ^:private metabase-index-prefix "mb_transform_idx_")--(defn- incremental-filter-index-name [schema table-name filter-column-name]- (let [prefix metabase-index-prefix- suffix (codecs/bytes->hex (buddy-hash/md5 (str/join "|" [schema table-name filter-column-name])))]- (str prefix suffix)))--(defn- should-index-incremental-filter-column? [driver database]- (and (driver.u/supports? driver :describe-indexes database)- (driver.u/supports? driver :transforms/index-ddl database)))--(defn- metabase-owned-index? [index]- (str/starts-with? (:index-name index) metabase-index-prefix))--(defn- metabase-incremental-filter-index [filter-column target]- (let [table-name (:name target)- schema (:schema target)- filter-name (:name filter-column)]- ;; match the schema used by describe-table-indexes (:value denotes the column name, assumed leading column if a composite index)- {:index-name (incremental-filter-index-name schema table-name filter-name)- :value filter-name}))--(defn- decide-secondary-index-ddl- "Decides which indexes should be dropped/created on the target table.-- e.g. Indexing the incremental :filter-column to accelerate MAX(target.checkpoint) queries on OLTP databases.-- Indexes are represented as maps with at least the keys :index-name, :value.- This matches the form used by [[metabase.driver/describe-table-indexes]].-- Returns a map:- :drop - a vector of indexes that are redundant and should be dropped.- :create - a vector of desirable indexes that do not exist and that should be created.-- Notes:- - If user covering indexes exist they should be reused.- - Will never drop user indexes.- - Indexes we previously created and are no longer required are dropped."- [{:keys [filter-column indexes target database]}]- (let [[mb-indexes user-indexes] ((juxt filter remove) metabase-owned-index? indexes)- default-mb-index (when filter-column (metabase-incremental-filter-index filter-column target))- column-name (:name filter-column)- existing-user-index (first (filter #(= (:value %) column-name) user-indexes))- existing-mb-index (first (filter #(= (:index-name %) (:index-name default-mb-index)) mb-indexes))- driver (:engine database)- intended-index (when (should-index-incremental-filter-column? driver database)- ;; Prefer reuse to not create redundant indexes- (or existing-user-index default-mb-index))- drop (if intended-index- (remove #(= (:index-name intended-index) (:index-name %)) mb-indexes)- mb-indexes)- create (when (and intended-index- (not= (:index-name intended-index) (:index-name existing-user-index))- (not= (:index-name intended-index) (:index-name existing-mb-index)))- [intended-index])]- {:drop (vec drop)- :create (vec create)}))--(defn execute-secondary-index-ddl-if-required!- "If target table index modifications are required, executes those CREATE/DROP commands.- See [[metabase.transforms-util/decide-secondary-index-ddl]] for details."- [transform run-id database target]- (when (driver.u/supports? (:engine database) :describe-indexes database)- (let [driver (:engine database)- indexes (driver/describe-table-indexes driver database target)- checkpoint (next-checkpoint transform)- {:keys [drop create]}- (decide-secondary-index-ddl- {:filter-column (:filter-column checkpoint)- :database database- :target target- :indexes indexes})]- (doseq [{:keys [index-name value]} drop]- (transforms.instrumentation/with-stage-timing [run-id [:import :drop-incremental-filter-index]]- (log/infof "Dropping secondary index %s(%s) for target %s" index-name value (pr-str target))- (driver/drop-index! driver (:id database) (:schema target) (:name target) index-name)))- (doseq [{:keys [index-name value]} create]- (transforms.instrumentation/with-stage-timing [run-id [:import :create-incremental-filter-index]]- (log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))- (driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))--(mu/defn handle-transform-complete!- "Handles followup tasks for when a transform has completed.-- Specifically, this syncs the target db, publishes a `:event/transform-run-complete` event, and potentially updates- the target table's index.-- See [[metabase.transforms-util/decide-secondary-index-ddl]] for details on the index handling."- [& {:keys [run-id transform db]}- :- [:map- [:run-id ::transforms.schema/run-id]- [:transform ::transforms.schema/transform]- [:db [:fn {:error/message "Must a t2 database object"} #(= (t2/model %) :model/Database)]]]]- (let [target (:target transform)]- (transforms.instrumentation/with-stage-timing [run-id [:import :table-sync]]- (when-let [table (sync-target! target db)]- (t2/update! :model/Table (:id table) {:transform_id (:id transform)}))- ;; This event must be published only after the sync is complete - the new table needs to be in AppDB.- (events/publish-event! :event/transform-run-complete- {:object {:db-id (:id db)- :transform-id (:id transform)- :transform-type (keyword (:type target))- :output-schema (:schema target)- :output-table (qualified-table-name (:engine db) target)}})- ;; Creating an index after sync means the filter column is known in the appdb.- ;; The index would be synced the next time sync runs, but at time of writing, index sync is disabled.- (execute-secondary-index-ddl-if-required! transform run-id db target))))diff --git a/src/metabase/transforms_base/core.clj b/src/metabase/transforms_base/core.cljnew file mode 100644index 00000000000..df3f8e654d7--- /dev/null+++ b/src/metabase/transforms_base/core.clj@@ -0,0 +1,46 @@+(ns metabase.transforms-base.core+ "Entry point for base transform execution.++ This namespace provides the main `execute!` function for running transforms+ without writing to transform_run rows. Results are returned in memory.++ For query transforms, this module handles execution directly.+ For python transforms, implementations are loaded separately via+ metabase-enterprise.transforms-python.base to avoid circular dependencies."+ (:require+ ;; Interface for dispatch+ [metabase.transforms-base.interface :as transforms-base.i]+ ;; Load query implementation - registers multimethods+ [metabase.transforms-base.query]+ [metabase.transforms-base.util :as transforms-base.u]))++(defn execute!+ "Execute transform and return result map without writing transform_run rows.++ This is the main entry point for base transform execution. Use this when you+ want to run a transform without database tracking (e.g., from workspaces).++ Performs the core execution via `execute-base!`, then runs post-processing+ (sync, events, secondary indexes) via `complete-execution!`.++ Options:+ - `:cancelled?` - (fn [] boolean), polled during execution to check for cancellation+ - `:run-id` - optional, for instrumentation/metrics (nil skips metrics recording)+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk] result)+ - `:publish-events?` - whether to publish Metabase events (default true)++ Returns a map:+ {:status :succeeded | :failed | :cancelled | :timeout+ :result <implementation-specific result>+ :error <exception if failed>+ :logs <string, for python transforms>}++ Target table sync (metabase_table writes) still occurs - this is necessary+ for the table to be usable after execution."+ ([transform]+ (execute! transform nil))+ ([transform opts]+ (let [result (transforms-base.i/execute-base! transform opts)]+ (when (= :succeeded (:status result))+ (transforms-base.u/complete-execution! transform opts))+ result)))diff --git a/src/metabase/transforms_base/init.clj b/src/metabase/transforms_base/init.cljnew file mode 100644index 00000000000..f5b5087792d--- /dev/null+++ b/src/metabase/transforms_base/init.clj@@ -0,0 +1,9 @@+(ns metabase.transforms-base.init+ "Loads multimethod implementations for the transforms-base module.++ `:query` implementations are loaded here directly.+ `:python` implementations are loaded via the enterprise init chain+ (metabase-enterprise.transforms-python.init)."+ (:require+ [metabase.transforms-base.ordering]+ [metabase.transforms-base.query]))diff --git a/src/metabase/transforms_base/interface.clj b/src/metabase/transforms_base/interface.cljnew file mode 100644index 00000000000..2efddae8174--- /dev/null+++ b/src/metabase/transforms_base/interface.clj@@ -0,0 +1,76 @@+(ns metabase.transforms-base.interface+ "Interface definitions for base transform execution.++ This namespace defines the multimethods that different transform types+ (query, python) must implement. The base module uses these for dispatch+ without any database writes for transform_run tracking.++ For scheduled execution with transform_run tracking, see metabase.transforms.interface.")++(defn transform->transform-type+ "Extract the transform type from a transform's source."+ [transform]+ (-> transform :source :type keyword))++(defmulti source-db-id+ "Return the ID of the source database for a given `transform`. The source database is where the data originates from+ before being transformed and written to the target destination.++ NOTE: Currently returns a single database ID, but we may want to change this to return multiple database IDs in the+ future since transforms can have multiple input sources."+ {:added "0.57.0" :arglists '([transform])}+ transform->transform-type)++(defmulti target-db-id+ "Return the ID of the target database for a given `transform`. The target database is the destination where the+ transformed data will be written. Often this is the same database as the source database."+ {:added "0.57.0" :arglists '([transform])}+ transform->transform-type)++(defmulti table-dependencies+ "Return a set of logical table dependencies of the transform, including indirect dependencies via cards.+ The transform execution system uses these dependencies to determine the correct order of execution+ and to detect circular dependencies.++ Each dependency is represented as one of:+ - `{:table <table-id>}`+ A dependency on a table that exists and has been synced.+ - `{:transform <transform-id>}`+ A dependency on a table that does not yet exist, but is known to be the target of another transform.+ Represents a 'placeholder' table (as we no table id / metadata) for the same purposes.+ - `{:table-ref {:database_id <db-id> :schema <schema> :table <name>}}`+ A dependency on a table by name, for cases where the table_id may not exist yet (e.g., when+ bulk importing transforms that depend on each other's outputs). The ordering system will+ resolve these to transforms by matching against target table definitions.++ An empty set indicates no dependencies."+ {:added "0.57.0" :arglists '([transform])}+ transform->transform-type)++(defmulti execute-base!+ "Execute a transform's core logic and return results.++ Does NOT:+ - Write transform_run rows+ - Sync target tables (caller should use `complete-execution!` or `transforms-base.core/execute!`)+ - Publish events (caller should use `complete-execution!` or `transforms-base.core/execute!`)++ Options:+ - `:cancelled?` - (fn [] boolean), polled during execution to check for cancellation+ - `:run-id` - optional, for instrumentation/metrics (nil skips metrics recording)+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk]) for timing instrumentation+ - `:message-log` - optional, pre-created message log atom (for python transforms)++ Returns a map:+ {:status :succeeded | :failed | :cancelled | :timeout+ :result <implementation-specific result>+ :error <exception if failed>+ :logs <string, for python transforms>}++ Implementations should:+ - Check `(cancelled?)` at safe points and return {:status :cancelled} if true+ - NOT create/update transform_run rows+ - NOT sync target tables or publish events (caller handles via complete-execution!)"+ {:added "0.57.0" :arglists '([transform options])}+ (fn [transform _options]+ (transform->transform-type transform)))diff --git a/src/metabase/transforms/ordering.clj b/src/metabase/transforms_base/ordering.cljsimilarity index 83%rename from src/metabase/transforms/ordering.cljrename to src/metabase/transforms_base/ordering.cljindex 654a1cb4a08..64b6998b614 100644--- a/src/metabase/transforms/ordering.clj+++ b/src/metabase/transforms_base/ordering.clj@@ -1,4 +1,7 @@-(ns metabase.transforms.ordering+(ns metabase.transforms-base.ordering+ "Transform dependency ordering and cycle detection.++ Pure functions for computing execution order based on table dependencies."(:require[clojure.set :as set][clojure.string :as str]@@ -10,36 +13,46 @@[metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.query-processor.preprocess :as qp.preprocess]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util.i18n :as i18n][metabase.util.malli :as mu]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (clojure.lang ExceptionInfo)))(set! *warn-on-reflection* true)+;;; ------------------------------------------------- Query Dependencies -------------------------------------------------++(defn query-table-dependencies+ "Compute table dependencies for a query transform.+ This is the base implementation - callers may wrap with additional error handling."+ [{:keys [source]}]+ (let [query (-> (:query source)+ transforms-base.u/massage-sql-query+ qp.preprocess/preprocess)+ driver (-> query+ lib.metadata/database+ :engine)]+ (if (lib/native-only-query? query)+ (driver/native-query-deps driver query)+ (into #{}+ (map (fn [table-id]+ {:table table-id}))+ (lib/all-source-table-ids query)))))+(defn- database-routing-error-ex-data [^Throwable e](when e(if (:database-routing-enabled (ex-data e))(ex-data e)(recur (.getCause e)))))-(defmethod transforms.i/table-dependencies :query- [{:keys [source] :as transform}]+(defmethod transforms-base.i/table-dependencies :query+ [transform](try- (let [query (-> (:query source)- transforms.util/massage-sql-query- qp.preprocess/preprocess)- driver (-> query- lib.metadata/database- :engine)]- (if (lib/native-only-query? query)- (driver/native-query-deps driver query)- (into #{}- (map (fn [table-id]- {:table table-id}))- (lib/all-source-table-ids query))))- (catch clojure.lang.ExceptionInfo e+ (query-table-dependencies transform)+ (catch ExceptionInfo e(if-some [data (database-routing-error-ex-data e)](let [message (i18n/trs "Failed to run transform because the database {0} has database routing turned on. Running transforms on databases with db routing enabled is not supported." (:database-name data))](throw (ex-info message@@ -49,9 +62,11 @@e)))(throw e)))))+;;; ------------------------------------------------- Ordering Logic -------------------------------------------------+(defn- dependency-map [transforms](into {}- (map (juxt :id transforms.i/table-dependencies))+ (map (juxt :id transforms-base.i/table-dependencies))transforms))(mu/defn- output-table-map@@ -95,7 +110,7 @@(let [;; Group all transforms by their database, skipping transforms with no target dbtransforms-by-db (->> transforms(keep (fn [transform]- (when-let [db-id (transforms.i/target-db-id transform)]+ (when-let [db-id (transforms-base.i/target-db-id transform)]{db-id [transform]})))(apply merge-with into))transform-ids (into #{} (map :id) transforms)@@ -167,7 +182,7 @@output-tables (output-table-map mp db-transforms)transform-ids (into #{} (map :id) db-transforms)target-refs (target-ref-map transforms)- node->children #(->> % transforms-by-id transforms.i/table-dependencies+ node->children #(->> % transforms-by-id transforms-base.i/table-dependencies(keep (fn [dep] (resolve-dependency dep output-tables transform-ids target-refs))))id->name (comp :name transforms-by-id)cycle (find-cycle node->children [transform-id])]diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.cljnew file mode 100644index 00000000000..f00c8fd41b9--- /dev/null+++ b/src/metabase/transforms_base/query.clj@@ -0,0 +1,138 @@+(ns metabase.transforms-base.query+ "Base query transform execution - core logic without transform_run tracking.++ This namespace handles MBQL/native query transform execution and returns+ results in memory rather than writing to transform_run rows."+ (:require+ [metabase.driver :as driver]+ [metabase.driver.util :as driver.u]+ [metabase.lib.schema.common :as schema.common]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.util.log :as log]+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++;;; ------------------------------------------------- Interface Implementations -------------------------------------------------++(defmethod transforms-base.i/source-db-id :query+ [transform]+ (-> transform :source :query :database))++(defmethod transforms-base.i/target-db-id :query+ [transform]+ ;; For query transforms, the target needs to match the source, so use the query as the source of truth.+ (or (-> transform :source :query :database)+ ;; Fallback to using a configured value.+ (get-in transform [:target :database])+ (:target_db_id transform)))++;;; ------------------------------------------------- Schemas -------------------------------------------------++(mr/def ::transform-details+ [:map+ [:transform-type [:enum {:decode/normalize schema.common/normalize-keyword} :table :table-incremental]]+ [:conn-spec :any]+ [:query ::qp.compile/compiled]+ [:output-table [:keyword {:decode/normalize schema.common/normalize-keyword}]]])++(mr/def ::transform-opts+ [:map+ [:overwrite? :boolean]])++;;; ------------------------------------------------- Helpers -------------------------------------------------++(defn- transform-opts [{:keys [transform-type]}]+ (case transform-type+ :table {:overwrite? true}+ ;; once we have more than just append, dispatch on :target-incremental-strategy+ :table-incremental {}))++;;; ------------------------------------------------- Base Execution -------------------------------------------------++(defn run-query-transform!+ "Execute query transform (MBQL/native). Returns result map.++ Does:+ - Compile query+ - Create schema if needed+ - Call driver/run-transform!++ Does NOT:+ - Create transform_run row+ - Update transform_run status+ - Sync target table (caller handles via complete-execution!)+ - Publish events (caller handles via complete-execution!)++ Options:+ - `:cancelled?` - (fn [] boolean), polled to check for cancellation+ - `:run-id` - optional, for instrumentation (nil skips metrics)+ - `:with-stage-timing-fn` - optional, (fn [run-id stage thunk] result)++ Returns:+ {:status :succeeded | :failed | :cancelled+ :result <driver result>+ :error <exception if failed>}"+ [{:keys [id source target] :as transform}+ {:keys [cancelled?] :as _opts}]+ (try+ ;; Check cancellation before starting+ (when (and cancelled? (cancelled?))+ (throw (ex-info "Transform cancelled before start" {:status :cancelled})))++ (let [db (get-in source [:query :database])+ {driver :engine :as database} (t2/select-one :model/Database db)+ _ (transforms-base.u/throw-if-db-routing-enabled! transform database)+ transform-details {:db-id db+ :database database+ :transform-id id+ :transform-type (keyword (:type target))+ :conn-spec (driver/connection-spec driver database)+ :query (transforms-base.u/compile-source transform)+ :output-schema (:schema target)+ :output-table (transforms-base.u/qualified-table-name driver target)}+ opts (transform-opts transform-details)+ features (transforms-base.u/required-database-features transform)]++ (when-not (every? (fn [feature] (driver.u/supports? (:engine database) feature database)) features)+ (throw (ex-info "The database does not support the requested transform target type."+ {:driver driver, :database database, :features features})))++ (log/info "Executing transform" id "with target" (pr-str target))++ ;; Create schema if needed+ (when-not (driver/schema-exists? driver db (:schema target))+ (driver/create-schema-if-needed! driver (:conn-spec transform-details) (:schema target)))++ ;; Check cancellation before running query+ (when (and cancelled? (cancelled?))+ (throw (ex-info "Transform cancelled before query execution" {:status :cancelled})))++ ;; Run the actual transform+ (let [result (driver/run-transform! driver transform-details opts)]++ ;; Check cancellation after query+ (when (and cancelled? (cancelled?))+ (throw (ex-info "Transform cancelled after query execution" {:status :cancelled})))++ {:status :succeeded+ :result result}))++ (catch Exception e+ (let [data (ex-data e)]+ (if (= :cancelled (:status data))+ {:status :cancelled+ :error e}+ (do+ (log/error e "Error executing transform")+ {:status :failed+ :error e}))))))++;;; ------------------------------------------------- Interface Implementation -------------------------------------------------++(defmethod transforms-base.i/execute-base! :query+ [transform opts]+ (run-query-transform! transform opts))diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljnew file mode 100644index 00000000000..3eef908348b--- /dev/null+++ b/src/metabase/transforms_base/util.clj@@ -0,0 +1,731 @@+(ns metabase.transforms-base.util+ "Shared utilities for transform execution.++ This namespace contains functions that are used by both query and python transforms+ and do NOT require transform_run database access."+ (:require+ [buddy.core.codecs :as codecs]+ [buddy.core.hash :as buddy-hash]+ [clojure.string :as str]+ [java-time.api :as t]+ [metabase.driver :as driver]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.driver.util :as driver.u]+ [metabase.events.core :as events]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.schema.common :as lib.schema.common]+ [metabase.query-processor :as qp]+ [metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.middleware.add-remaps :as remap]+ [metabase.query-processor.middleware.catch-exceptions :as qp.catch-exceptions]+ [metabase.query-processor.parameters.dates :as params.dates]+ [metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.sync.core :as sync]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.util :as u]+ [metabase.util.date-2 :as u.date]+ [metabase.util.honey-sql-2 :as h2x]+ [metabase.util.i18n :as i18n]+ [metabase.util.log :as log]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [toucan2.core :as t2])+ (:import+ (java.time Instant LocalDate LocalDateTime LocalTime OffsetDateTime OffsetTime ZonedDateTime)+ (java.util Date)))++(set! *warn-on-reflection* true)++;;; ------------------------------------------------- Constants -------------------------------------------------++(def ^:const transform-temp-table-prefix+ "Prefix used for temporary tables created during transform execution."+ "mb_transform_temp_table")++;;; ------------------------------------------------- Transform Type Predicates -------------------------------------------------++(defn transform-type+ "Get the type of a transform"+ [transform]+ (-> transform :source :type keyword))++(defn query-transform?+ "Check if this is a query transform: native query / mbql query."+ [transform]+ (= :query (transform-type transform)))++(defn native-query-transform?+ "Check if this is a native query transform.+ Note: The transform should be normalized (via `normalize-transform`) before calling this function."+ [transform]+ (when (query-transform? transform)+ (let [query (-> transform :source :query)]+ (lib/native-only-query? query))))++(defn python-transform?+ "Check if this is a Python transform."+ [transform]+ (= :python (transform-type transform)))++(defn transform-source-database+ "Get the source database from a transform"+ [transform]+ (case (transform-type transform)+ :query (-> transform :source :query :database)+ :python (-> transform :source :source-database)))++;;; ------------------------------------------------- Transform Normalization -------------------------------------------------++(defn normalize-transform+ "Normalize a transform's source query, similar to how transforms are normalized when read from the database.+ This should be called on transforms before processing them to ensure queries are in the expected format."+ [transform]+ (if (and (map? transform)+ (= (:type transform) "transform")+ (get-in transform [:source :query]))+ (update-in transform [:source :query] lib-be/normalize-query)+ transform))++(defn transform-source-type+ "Returns the type of a transform's source: :python, :native, or :mbql.+ Throws if the source type cannot be detected.+ Note: The transform should be normalized (via `normalize-transform`) before calling this function."+ [source]+ (case (keyword (:type source))+ :python :python+ :query (if (lib/native-only-query? (:query source))+ :native+ :mbql)+ (throw (ex-info (str "Unknown transform source type: " (:type source))+ {:source source}))))++;;; ------------------------------------------------- Feature Checks -------------------------------------------------++(defn required-database-features+ "Returns the database features necessary to execute `transform`."+ [transform]+ (if (python-transform? transform)+ [:transforms/python]+ [:transforms/table]))++;;; ------------------------------------------------- Table Names -------------------------------------------------++(defn qualified-table-name+ "Return the name of the target table of a transform as a possibly qualified symbol."+ [_driver {:keys [schema name]}]+ (if schema+ (keyword schema name)+ (keyword name)))++(defn temp-table-name+ "Generate a temporary table name with current timestamp in milliseconds.+ If table name would exceed max table name length for the driver, fallback to using a shorter timestamp"+ [driver schema]+ (let [max-len (max 1 (or (driver/table-name-length-limit driver) Integer/MAX_VALUE))+ timestamp (str (System/currentTimeMillis))+ prefix (str transform-temp-table-prefix "_")+ available (- max-len (count prefix))+ ;; If we don't have enough space, take the later digits of the timestamp+ suffix (if (>= available (count timestamp))+ timestamp+ (subs timestamp (- (count timestamp) available)))+ table-name (str prefix suffix)]+ (keyword schema table-name)))++;;; ------------------------------------------------- Query Helpers -------------------------------------------------++(defn massage-sql-query+ "Adjusts mbql query for use in a transform."+ [query]+ (-> query+ remap/disable-remaps+ lib/disable-default-limit))++;;; ------------------------------------------------- Incremental/Checkpoint Helpers -------------------------------------------------++(defn- checkpoint-incremental?+ "Returns true if `source` uses checkpoint-based incremental strategy."+ [source]+ (= :checkpoint (some-> source :source-incremental-strategy :type keyword)))++(defn supported-incremental-filter-type?+ "Returns true if the given base-type is supported for incremental filtering.++ We only support temporal (timestamp/tz) and numeric (int/float) types."+ [base-type]+ (or (isa? base-type :type/Temporal)+ (isa? base-type :type/Number)))++(defn- source->checkpoint-filter-unique-key+ "Extract the checkpoint filter column from `query` using the unique key specified in `source-incremental-strategy`."+ [query source-incremental-strategy]+ (some->> source-incremental-strategy :checkpoint-filter-unique-key (lib/column-with-unique-key query)))++(defn- source->checkpoint-filter-column+ "Resolve the checkpoint filter column for an incremental transform.++ Tries to resolve the column using the unique key first.+ Falls back to looking up the column by name from the target table if a `:checkpoint-filter` is specified.++ Validates that the resolved column has a supported type for checkpoint filtering (numeric or temporal).+ Throws an exception if the column type is not supported."+ [query source-incremental-strategy table metadata-provider]+ (let [{:keys [checkpoint-filter checkpoint-filter-unique-key]} source-incremental-strategy]+ (when-some [{column-name :name+ :keys [base-type]+ :as column}+ (cond+ checkpoint-filter-unique-key+ (source->checkpoint-filter-unique-key query source-incremental-strategy)+ checkpoint-filter+ (when-some [field-id (t2/select-one-pk :model/Field+ :table_id (:id table)+ :name checkpoint-filter)]+ (lib.metadata/field metadata-provider field-id)))]+ (when-not (supported-incremental-filter-type? base-type)+ (throw (ex-info (str "Checkpoint column '" column-name "' has unsupported type " (pr-str base-type) ". "+ "Only numeric and temporal columns are supported for incremental filtering.")+ {:column-name column-name+ :base-type base-type})))+ column)))++(declare target-table)++(defn next-checkpoint+ "Build a query to compute the MAX of the checkpoint column from the target table.++ Returns a map with `:query` (MBQL query selecting the max) and `:filter-column` (column metadata),+ or `nil` if the transform doesn't use checkpoint-based incremental strategy or the target table doesn't exist."+ [{:keys [source target] :as transform}]+ (let [db-id (transforms-base.i/target-db-id transform)]+ (when (checkpoint-incremental? source)+ (when-let [table (target-table db-id target)]+ (let [metadata-provider (lib-be/application-database-metadata-provider db-id)+ table-metadata (lib.metadata/table metadata-provider (:id table))+ query (lib/query metadata-provider table-metadata)]+ (when-let [filter-column (source->checkpoint-filter-column query+ (:source-incremental-strategy source)+ table metadata-provider)]+ {:query (-> query (lib/aggregate (lib/max filter-column)))+ :filter-column filter-column}))))))++(defn- next-checkpoint-value+ "Execute the checkpoint query and normalize the result for database insertion.+ Returns `nil` if the target table is empty."+ [{:keys [query filter-column]}]+ (let [{:keys [base-type]} filter-column+ v (some-> query qp/process-query :data :rows first first)]+ ;; QP return values are lossy, we do a bit of parsing to ensure they're of the right+ ;; shape for reinsertion+ (cond+ (nil? v)+ nil++ (isa? base-type :type/Integer)+ (bigint v)++ ;; any other number that's not an integer, should be a decimal/float+ (number? v)+ (bigdec v)++ :else v)))++(defn preprocess-incremental-query+ "Add checkpoint checkpoint filtering to a query for incremental execution.++ For native queries with a `checkpoint` template tag, adds the checkpoint as a parameter.+ For MBQL queries, adds a filter clause `WHERE checkpoint_column > checkpoint`.+ Returns the query unchanged on first run (no checkpoint) or for native queries without the checkpoint tag."+ [query source-incremental-strategy checkpoint]+ (if-let [checkpoint-value (next-checkpoint-value checkpoint)]+ (if (lib/native? query)+ ;; native query with explicit checkpoint filter+ (if (get-in query [:stages 0 :template-tags "checkpoint"])+ (update query :parameters conj+ {:type (if (number? checkpoint-value) :number :text)+ :target [:variable [:template-tag "checkpoint"]]+ :value checkpoint-value})+ query)+ ;; mbql query+ (lib/filter query (lib/> (source->checkpoint-filter-unique-key query source-incremental-strategy) checkpoint-value)))+ query))++(defn- post-process-incremental-query+ "Wrap a compiled native query with checkpoint filtering for native queries without explicit checkpoint tags.++ Generates SQL that wraps the original query as a subquery and filters by `checkpoint_filter > (checkpoint_query)`. "+ [outer-query driver {:keys [source-incremental-strategy] :as source} {checkpoint-query :query :as checkpoint}]+ (let [{:keys [checkpoint-filter]} source-incremental-strategy]+ (if (and (lib/native? (:query source))+ (not (get-in (:query source) [:stages 0 :template-tags "checkpoint"]))+ (next-checkpoint-value checkpoint))+ (let [wrap-query (fn [query]+ (let [honeysql-query {:select [:*]+ :from [[[:raw (str "(" query ")")] :subquery]]+ :where [:> (h2x/identifier :field checkpoint-filter)+ [:raw (str "(" (:query (qp.compile/compile checkpoint-query)) ")")]]}]+ (first (sql.qp/format-honeysql driver honeysql-query))))]+ (update outer-query :query wrap-query))+ outer-query)))++(mu/defn validate-transform-query :- [:maybe [:map [:error :string]]]+ "Verifies that a query transform's query can actually be run as is. Returns nil on success and an error map on failure."+ [{:keys [source]}]+ (case (keyword (:type source))+ :query+ (try+ (qp.preprocess/preprocess (:query source))+ nil+ (catch Exception e+ (qp.catch-exceptions/exception-response e)))))++(defn compile-source+ "Compile the source query of a transform to SQL, applying incremental filtering if required."+ [{:keys [source] :as transform}]+ (let [{:keys [source-incremental-strategy] query-type :type} source]+ (case (keyword query-type)+ :query+ (let [checkpoint (next-checkpoint transform)+ query (:query source)+ driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]+ (binding [driver/*compile-with-inline-parameters*+ (or (= :clickhouse driver)+ driver/*compile-with-inline-parameters*)]+ (-> query+ (preprocess-incremental-query source-incremental-strategy checkpoint)+ massage-sql-query+ qp.compile/compile+ (post-process-incremental-query driver source checkpoint)))))))++;;; ------------------------------------------------- Target Table Management -------------------------------------------------++(defn target-table+ "Load the `target` table of a transform from the database specified by `database-id`."+ [database-id target & kv-args]+ (some-> (apply t2/select-one :model/Table+ :db_id database-id+ :schema (:schema target)+ :name (:name target)+ kv-args)+ (t2/hydrate :db)))++(defn target-table-exists?+ "Test if the target table of a transform already exists."+ [{:keys [target] :as transform}]+ (let [db-id (transforms-base.i/target-db-id transform)+ {driver :engine :as database} (t2/select-one :model/Database db-id)]+ (driver/table-exists? driver database target)))++(defn- sync-table!+ ([database target] (sync-table! database target nil))+ ([database target {:keys [create?]}]+ (when-let [table (or (target-table (:id database) target)+ (when create?+ (sync/create-table! database (select-keys target [:schema :name :data_source :data_authority]))))]+ (sync/sync-table! table)+ table)))++(defn activate-table-and-mark-computed!+ "Activate table for `target` in `database` in the app db."+ [database target]+ (when-let [table (sync-table! database (assoc target+ :data_authority :computed+ :data_source :metabase-transform)+ {:create? true})]+ (when-not (:active table)+ (t2/update! :model/Table (:id table) {:active true}))+ table))++(defn sync-target!+ "Sync target of a transform"+ [target database]+ ;; sync the new table (note that even a failed sync status means that the execution succeeded)+ (log/info "Syncing target" (pr-str target) "for transform")+ (activate-table-and-mark-computed! database target))++(defn deactivate-table!+ "Deactivate table for `target` in `database` in the app db."+ [database target]+ (when-let [table (sync-table! database target)]+ ;; TODO this should probably be a function in the sync module+ (t2/update! :model/Table (:id table) {:active false})))++(defn delete-target-table!+ "Delete the target table of a transform and sync it from the app db."+ [{:keys [id target], :as transform}]+ (when target+ (let [target (update target :type keyword)+ database-id (transforms-base.i/target-db-id transform)]+ (when database-id+ (if-let [{driver :engine :as database} (t2/select-one :model/Database database-id)]+ (do+ (driver/drop-transform-target! driver database target)+ (log/info "Deactivating target " (pr-str target) "for transform" id)+ (deactivate-table! database target))+ (log/warnf "Skipping drop of transform target %s for transform %d: database %d not found"+ (pr-str target) id database-id))))))++(defn delete-target-table-by-id!+ "Delete the target table of the transform specified by `transform-id`."+ [transform-id]+ (delete-target-table! (t2/select-one :model/Transform transform-id)))++;;; ------------------------------------------------- Table DDL -------------------------------------------------++(mr/def ::column-definition+ [:map+ [:name :string]+ [:type ::lib.schema.common/base-type]+ [:nullable? {:optional true} :boolean]])++(mr/def ::table-definition+ [:map+ [:name :keyword]+ [:columns [:sequential ::column-definition]]+ [:primary-key {:optional true} [:sequential :string]]])++(mu/defn create-table-from-schema!+ "Create a table from a table-schema"+ [driver :- :keyword+ database-id :- pos-int?+ table-schema :- ::table-definition]+ (let [{:keys [columns] table-name :name} table-schema+ column-definitions (mapv (fn [{:keys [name type database-type]}]+ (let [db-type (if database-type+ [[:raw database-type]]+ (try+ (driver/type->database-type driver type)+ (catch IllegalArgumentException _+ (log/warnf "Couldn't determine database type for type %s, fallback to Text" type)+ (driver/type->database-type driver :type/Text))))]+ [name db-type]))+ columns)+ primary-key-opts (select-keys table-schema [:primary-key])]+ (log/infof "Creating table %s with %d columns" table-name (count columns))+ (driver/create-table! driver database-id table-name column-definitions primary-key-opts)))++(defn drop-table!+ "Drop a table in the database."+ [driver database-id table-name]+ (log/infof "Dropping table %s" table-name)+ (driver/drop-table! driver database-id table-name))++(defn rename-tables!+ "Rename multiple tables atomically within a transaction using the new driver/rename-tables method.+ This is a simpler, composable operation that only handles renaming."+ [driver database-id rename-map]+ (log/infof "Renaming tables: %s" (pr-str rename-map))+ (driver/rename-tables! driver database-id rename-map))++;;; ------------------------------------------------- Secondary Index Management -------------------------------------------------++(def ^:private metabase-index-prefix "mb_transform_idx_")++(defn- incremental-filter-index-name [schema table-name filter-column-name]+ (let [prefix metabase-index-prefix+ suffix (codecs/bytes->hex (buddy-hash/md5 (str/join "|" [schema table-name filter-column-name])))]+ (str prefix suffix)))++(defn- should-index-incremental-filter-column? [driver database]+ (and (driver.u/supports? driver :describe-indexes database)+ (driver.u/supports? driver :transforms/index-ddl database)))++(defn- metabase-owned-index? [index]+ (str/starts-with? (:index-name index) metabase-index-prefix))++(defn- metabase-incremental-filter-index [filter-column target]+ (let [table-name (:name target)+ schema (:schema target)+ filter-name (:name filter-column)]+ ;; match the schema used by describe-table-indexes (:value denotes the column name, assumed leading column if a composite index)+ {:index-name (incremental-filter-index-name schema table-name filter-name)+ :value filter-name}))++(defn- decide-secondary-index-ddl+ "Decides which indexes should be dropped/created on the target table.++ e.g. Indexing the incremental :filter-column to accelerate MAX(target.checkpoint) queries on OLTP databases.++ Indexes are represented as maps with at least the keys :index-name, :value.+ This matches the form used by [[metabase.driver/describe-table-indexes]].++ Returns a map:+ :drop - a vector of indexes that are redundant and should be dropped.+ :create - a vector of desirable indexes that do not exist and that should be created.++ Notes:+ - If user covering indexes exist they should be reused.+ - Will never drop user indexes.+ - Indexes we previously created and are no longer required are dropped."+ [{:keys [filter-column indexes target database]}]+ (let [[mb-indexes user-indexes] ((juxt filter remove) metabase-owned-index? indexes)+ default-mb-index (when filter-column (metabase-incremental-filter-index filter-column target))+ column-name (:name filter-column)+ existing-user-index (first (filter #(= (:value %) column-name) user-indexes))+ existing-mb-index (first (filter #(= (:index-name %) (:index-name default-mb-index)) mb-indexes))+ driver (:engine database)+ intended-index (when (should-index-incremental-filter-column? driver database)+ ;; Prefer reuse to not create redundant indexes+ (or existing-user-index default-mb-index))+ drop (if intended-index+ (remove #(= (:index-name intended-index) (:index-name %)) mb-indexes)+ mb-indexes)+ create (when (and intended-index+ (not= (:index-name intended-index) (:index-name existing-user-index))+ (not= (:index-name intended-index) (:index-name existing-mb-index)))+ [intended-index])]+ {:drop (vec drop)+ :create (vec create)}))++(defn execute-secondary-index-ddl-if-required!+ "If target table index modifications are required, executes those CREATE/DROP commands.+ See [[decide-secondary-index-ddl]] for details.++ `run-id` is optional - if nil, instrumentation/timing is skipped.+ `with-stage-timing-fn` is optional - if provided, will be called as (with-stage-timing-fn run-id stage thunk)."+ ([transform database target]+ (execute-secondary-index-ddl-if-required! transform nil database target nil))+ ([transform run-id database target with-stage-timing-fn]+ (when (driver.u/supports? (:engine database) :describe-indexes database)+ (let [driver (:engine database)+ indexes (driver/describe-table-indexes driver database target)+ checkpoint (next-checkpoint transform)+ {:keys [drop create]}+ (decide-secondary-index-ddl+ {:filter-column (:filter-column checkpoint)+ :database database+ :target target+ :indexes indexes})+ maybe-timed (fn [stage thunk]+ (if (and run-id with-stage-timing-fn)+ (with-stage-timing-fn run-id stage thunk)+ (thunk)))]+ (doseq [{:keys [index-name value]} drop]+ (maybe-timed [:import :drop-incremental-filter-index]+ (fn []+ (log/infof "Dropping secondary index %s(%s) for target %s" index-name value (pr-str target))+ (driver/drop-index! driver (:id database) (:schema target) (:name target) index-name))))+ (doseq [{:keys [index-name value]} create]+ (maybe-timed [:import :create-incremental-filter-index]+ (fn []+ (log/infof "Creating secondary index %s(%s) for target %s" index-name value (pr-str target))+ (driver/create-index! driver (:id database) (:schema target) (:name target) index-name [value]))))))))++;;; ------------------------------------------------- Post-Execution Completion -------------------------------------------------++(defn complete-execution!+ "Post-processing steps after a transform has been executed successfully.++ Performs:+ - Sync target table to AppDB+ - Set `transform_id` on the target table+ - Publish Metabase events (unless `:publish-events?` is false)+ - Create/drop secondary indexes++ This is called after the core execution completes. Callers that use+ `run-cancelable-transform!` should call this AFTER `succeed-started-run!`+ to preserve the correct order of operations."+ [transform opts]+ (let [{:keys [target]} transform+ {:keys [run-id with-stage-timing-fn publish-events?]+ :or {publish-events? true}} opts+ db-id (transforms-base.i/target-db-id transform)+ database (t2/select-one :model/Database db-id)]+ ;; Sync target table+ (sync-target! target database)+ ;; Mark the table as owned by this transform+ (when-let [table (t2/select-one :model/Table+ :db_id db-id+ :schema (:schema target)+ :name (:name target))]+ (t2/update! :model/Table (:id table) {:transform_id (:id transform)}))+ ;; Publish event after sync so the table exists in AppDB.+ (when publish-events?+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id db-id+ :transform-id (:id transform)+ :transform-type (keyword (:type target))+ :output-schema (:schema target)+ :output-table (qualified-table-name (:engine database) target)}}))+ ;; Create secondary indexes if needed+ (execute-secondary-index-ddl-if-required!+ transform run-id database target with-stage-timing-fn)))++;;; ------------------------------------------------- Source Table Resolution -------------------------------------------------++(def ^:private ^:const batch-lookup-chunk-size+ "Maximum number of table refs to query at once to avoid SQL query size limits."+ 100)++(defn batch-lookup-table-ids+ "Batch lookup table IDs from ref maps. Returns {[db_id schema name] -> table_id}.+ Queries the exact conjunction of each [database_id, schema, table] triple rather than+ a Cartesian product of all values. Uses chunking to avoid query size limits."+ [refs]+ (when (seq refs)+ (let [unique-refs (distinct (map (juxt :database_id :schema :table) refs))+ ref->clause (fn [[db-id schema table-name]]+ [:and+ [:= :db_id db-id]+ (if (some? schema)+ [:= :schema schema]+ [:is :schema nil])+ [:= :name table-name]])+ fetch-batch (fn [batch]+ (t2/select-fn->fn (juxt :db_id :schema :name) :id+ [:model/Table :id :db_id :schema :name]+ {:where (into [:or] (map ref->clause batch))}))]+ (into {} (mapcat fetch-batch) (partition-all batch-lookup-chunk-size unique-refs)))))++(defn- source-table-ref->key+ "Convert a source table ref map to a lookup key [db_id schema name]."+ [{:keys [database_id schema table]}]+ [database_id schema table])++(defn- missing-table-id?+ "Returns true if `v` is a source table ref map that needs table_id lookup."+ [v]+ (and (map? v) (nil? (:table_id v))))++(defn normalize-source-tables+ "Normalize source-tables to consistent map format {:database_id :schema :table :table_id}.++ The old format stored just integer table IDs. New transforms store maps on write.+ Old data is converted on read via transform-source-out for backwards compatibility.++ Throws if an integer table ID references a non-existent table.+ Map refs with non-existent tables get nil table_id (resolved later at execute time)."+ [source-tables]+ (let [int-table-ids (into #{} (filter int?) (vals source-tables))+ int-id->metadata (when (seq int-table-ids)+ (t2/select-pk->fn (fn [{:keys [db_id schema name]}]+ {:database_id db_id :schema schema :table name})+ [:model/Table :id :db_id :schema :name]+ :id [:in int-table-ids]))+ missing-ids (when (seq int-table-ids)+ (remove int-id->metadata int-table-ids))+ refs-needing-id (filter missing-table-id? (vals source-tables))+ ref-lookup (or (batch-lookup-table-ids refs-needing-id) {})]+ (when (seq missing-ids)+ (throw (ex-info (str "Tables not found for ids: " (str/join ", " (sort missing-ids)))+ {:table_ids (vec missing-ids)})))+ (update-vals source-tables+ (fn [v]+ (cond+ (int? v) (assoc (int-id->metadata v) :table_id v)+ (:table_id v) v+ :else (assoc v :table_id (ref-lookup (source-table-ref->key v))))))))++(defn resolve-source-tables+ "Resolve source-tables to {alias -> table_id}. Throws if any table not found.+ For execute time - all entries must resolve to valid table IDs.+ Handles both integer IDs (old format) and map refs (new format)."+ [source-tables]+ (let [needs-lookup (filter missing-table-id? (vals source-tables))+ lookup (or (batch-lookup-table-ids needs-lookup) {})+ resolved (u/for-map [[alias v] source-tables]+ [alias (if (int? v)+ v+ (or (:table_id v) (lookup (source-table-ref->key v))))])+ unresolved (for [[alias table-id] resolved+ :when (nil? table-id)+ :let [v (get source-tables alias)]]+ {:alias alias+ :table (if-let [schema (:schema v)]+ (str schema "." (:table v))+ (:table v))+ :ref v})]+ (when (seq unresolved)+ (throw (ex-info (str "Tables not found: " (str/join ", " (map :table unresolved)))+ {:unresolved unresolved+ :transform-message "Input table not found"})))+ resolved))++;;; ------------------------------------------------- Timestamp Helpers -------------------------------------------------++(defn ->instant+ "Convert a temporal value `t` to an Instant in the system timezone."+ ^Instant [t]+ (when t+ (condp instance? t+ Instant t+ Date (.toInstant ^Date t)+ OffsetDateTime (.toInstant ^OffsetDateTime t)+ ZonedDateTime (.toInstant ^ZonedDateTime t)+ LocalDateTime (recur (.atZone ^LocalDateTime t (t/zone-id)))+ String (recur (u.date/parse t))+ LocalTime (recur (.atDate ^LocalTime t (t/local-date)))+ OffsetTime (recur (.atDate ^OffsetTime t (t/local-date)))+ LocalDate (recur (.atStartOfDay ^LocalDate t))+ (throw (ex-info (str "Cannot convert temporal " t " of type " (type t) " to an Instant")+ {:temporal t})))))++(defn utc-timestamp-string+ "Convert the timestamp t to a string encoding the it in the system timezone."+ [t]+ (-> t ->instant str))++(defn localize-run-timestamps+ "Convert the timestamps of a `run` to ISO strings in UTC."+ [run]+ (-> run+ (u/update-some :start_time utc-timestamp-string)+ (u/update-some :end_time utc-timestamp-string)))++;;; ------------------------------------------------- Filter Transforms -------------------------------------------------++(defn- matching-timestamp?+ [job field-path {:keys [start end]}]+ (when-let [field-instant (->instant (get-in job field-path))]+ (let [parse #(-> % u.date/parse ->instant)]+ ;; logic here is to find when it's not matching and invert this+ (not (or (and start (.isBefore field-instant (parse start)))+ (and end (.isAfter field-instant (parse end))))))))++(defn ->date-field-filter-xf+ "Returns an xform for a date filter."+ [field-path filter-value]+ (if-let [range (some-> filter-value (params.dates/date-string->range {:inclusive-end? false}))]+ (filter #(matching-timestamp? % field-path range))+ identity))++(defn ->status-filter-xf+ "Returns an xform for a transform run status filter."+ [field-path statuses]+ (if-let [statuses (->> statuses (map keyword) set not-empty)]+ (filter #(statuses (get-in % field-path)))+ identity))++(defn ->tag-filter-xf+ "Returns an xform for a transform tag filter."+ [field-path tag-ids]+ (if-let [tag-ids (-> tag-ids set not-empty)]+ (filter #(some tag-ids (get-in % field-path)))+ identity))++;;; ------------------------------------------------- Misc -------------------------------------------------++(defn is-temp-transform-table?+ "Return true when `table` matches the transform temporary table naming pattern."+ [table]+ (when-let [table-name (:name table)]+ (str/starts-with? (u/lower-case-en table-name) transform-temp-table-prefix)))++(defn db-routing-enabled?+ "Returns whether or not the given database is either a router or destination database"+ [db-or-id]+ (or (t2/exists? :model/DatabaseRouter :database_id (u/the-id db-or-id))+ (some->> (:router-database-id db-or-id)+ (t2/exists? :model/DatabaseRouter :database_id))))++(defn throw-if-db-routing-enabled!+ "Throws if the database has routing enabled. Call before any driver operations to get a+ clear error message rather than a confusing driver-level failure."+ [transform database]+ (when (db-routing-enabled? database)+ (throw (ex-info (i18n/tru "Failed to run the transform ({0}) because the database ({1}) has database routing turned on. Running transforms on databases with db routing enabled is not supported."+ (:name transform)+ (:name database))+ {:driver (:engine database), :database database}))))diff --git a/src/metabase/transforms_inspector/context.clj b/src/metabase/transforms_inspector/context.cljindex 28672555bf7..27d6df18c56 100644--- a/src/metabase/transforms_inspector/context.clj+++ b/src/metabase/transforms_inspector/context.clj@@ -6,10 +6,10 @@[metabase.driver :as driver][metabase.lib.core :as lib][metabase.query-processor.preprocess :as qp.preprocess]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms-inspector.query-analysis :as query-analysis][metabase.transforms-inspector.schema :as transforms-inspector.schema]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.util :as transforms.util][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]@@ -30,13 +30,13 @@"Extract source table information for a transform.Returns a seq of maps with :table-id, :table-name, :schema, and :db-id."{:arglists '([transform])}- (fn [transform] (transforms.util/transform-source-type (:source transform))))+ (fn [transform] (transforms-base.u/transform-source-type (:source transform))))(defmethod extract-sources :mbql[{:keys [source]}](try(let [query (-> (:query source)- transforms.util/massage-sql-query+ transforms-base.u/massage-sql-queryqp.preprocess/preprocess)table-ids (lib/all-source-table-ids query)](table-ids->source-info table-ids))@@ -48,9 +48,9 @@[{:keys [source] :as transform}](try(let [query (-> (:query source)- transforms.util/massage-sql-query+ transforms-base.u/massage-sql-queryqp.preprocess/preprocess)- db-id (transforms.util/transform-source-database transform)+ db-id (transforms-base.u/transform-source-database transform)driver (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)deps (driver/native-query-deps driver query)table-ids (keep :table deps)]@@ -63,7 +63,7 @@[transform](try(let [source-tables (get-in transform [:source :source-tables])- normalized (transforms.util/normalize-source-tables source-tables)+ normalized (transforms-base.u/normalize-source-tables source-tables)table-ids (keep (fn [[_ v]] (:table_id v)) normalized)](table-ids->source-info table-ids))(catch Exception e@@ -79,8 +79,8 @@(defn- get-target-table"Get the target table for a transform. Returns nil if the target doesn't exist."[{:keys [target] :as transform}]- (let [db-id (transforms.i/target-db-id transform)]- (transforms.util/target-table db-id target)))+ (let [db-id (transforms-base.i/target-db-id transform)]+ (transforms-base.u/target-table db-id target)));;; -------------------------------------------------- Field Metadata --------------------------------------------------@@ -250,7 +250,7 @@(mu/defn build-context :- ::context"Build context for lens discovery and generation."[transform]- (let [source-type (transforms.util/transform-source-type (:source transform))+ (let [source-type (transforms-base.u/transform-source-type (:source transform))sources-info (mapv build-table-info (extract-sources transform))target-table (get-target-table transform)target-info (when target-table@@ -259,7 +259,7 @@join-structure (:join-structure query-info)column-matches (when (and (seq sources-info) target-info)(seq (match-columns sources-info target-info query-info)))- db-id (transforms.util/transform-source-database transform)]+ db-id (transforms-base.u/transform-source-database transform)](cond-> {:source-type source-type:sources sources-info:target target-infodiff --git a/src/metabase/transforms_inspector/query_analysis.clj b/src/metabase/transforms_inspector/query_analysis.cljindex 7b51591e138..746e3f22b6d 100644--- a/src/metabase/transforms_inspector/query_analysis.clj+++ b/src/metabase/transforms_inspector/query_analysis.clj@@ -17,7 +17,7 @@[metabase.lib.core :as lib][metabase.query-processor.preprocess :as qp.preprocess][metabase.sql-tools.core :as sql-tools]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms-base.util :as transforms-base.u][metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.log :as log]@@ -81,7 +81,7 @@Returns {:join-structure [...] :visited-fields {...}} or nil on failure."[transform](let [preprocessed (-> transform :source :query- transforms.util/massage-sql-query+ transforms-base.u/massage-sql-queryqp.preprocess/preprocess)](when (<= (count (:stages preprocessed)) 1){:preprocessed-query preprocessed@@ -232,10 +232,10 @@[transform sources](try(let [native-query (-> (get-in transform [:source :query])- transforms.util/massage-sql-query+ transforms-base.u/massage-sql-queryqp.preprocess/preprocess)sql (lib/raw-native-query native-query)- db-id (transforms.util/transform-source-database transform)+ db-id (transforms-base.u/transform-source-database transform)driver (t2/select-one-fn (comp keyword :engine) :model/Database :id db-id)parsed (macaw-parsed-query sql driver)ast (macaw.ast/->ast parsed {:with-instance? false})]diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 9a2ed58498c..29a1523e572 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -13,13 +13,14 @@[metabase.request.core :as request][metabase.server.core :as server][metabase.sql-tools.core :as sql-tools]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms-inspector.core :as inspector][metabase.transforms-inspector.schema :as inspector.schema][metabase.transforms-rest.api.transform-job][metabase.transforms-rest.api.transform-tag][metabase.transforms.core :as transforms.core][metabase.transforms.schema :as transforms.schema]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms.util :as transforms.u][metabase.util.i18n :refer [deferred-tru]][metabase.util.jvm :as u.jvm][metabase.util.malli.registry :as mr]@@ -169,12 +170,12 @@(transforms.core/check-feature-enabled! body)(transforms.core/validate-incremental-column-type! body)- (api/check (not (transforms.util/target-table-exists? body))+ (api/check (not (transforms-base.u/target-table-exists? body))403(deferred-tru "A table with that name already exists."))(-> (transforms.core/create-transform! body)transforms.core/python-source-table-ref->table-id- transforms.util/add-source-readable))+ transforms.u/add-source-readable))(api.macros/defendpoint :get "/:id" :- TransformResponse"Get a specific transform."@@ -193,7 +194,7 @@dependencies (map id->transform dep-ids)](->> (t2/hydrate dependencies :creator :owner)(mapv transforms.core/python-source-table-ref->table-id)- transforms.util/add-source-readable)))+ transforms.u/add-source-readable)))(def ^:private MergeHistoryEntry[:map@@ -244,7 +245,7 @@(-> (transforms.core/paged-runs (assoc query-params:offset (request/offset):limit (request/limit)))- (update :data #(map transforms.util/localize-run-timestamps %))))+ (update :data #(map transforms-base.u/localize-run-timestamps %))))(api.macros/defendpoint :get "/run/:run-id" :- TransformRunResponse"Get a transform run by ID."@@ -253,7 +254,7 @@(api/check-data-analyst)(let [run (api/check-404 (t2/select-one :model/TransformRun :id run-id))](-> (t2/hydrate run [:transform :collection :transform_tag_ids])- transforms.util/localize-run-timestamps)))+ transforms-base.u/localize-run-timestamps)))(api.macros/defendpoint :put "/:id" :- TransformResponse"Update a transform."@@ -284,7 +285,7 @@[{:keys [id]} :- [:map[:id ms/PositiveInt]]](api/write-check :model/Transform id)- (transforms.util/delete-target-table-by-id! id)+ (transforms-base.u/delete-target-table-by-id! id)nil)(api.macros/defendpoint :post "/:id/cancel" :- :nil@@ -294,7 +295,7 @@(let [transform (api/write-check :model/Transform id)run (api/check-404 (transforms.core/running-run-for-transform-id id))](transforms.core/mark-cancel-started-run! (:id run))- (when (transforms.util/python-transform? transform)+ (when (transforms-base.u/python-transform? transform)(transforms.core/cancel-run! (:id run))))nil)diff --git a/src/metabase/transforms_rest/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.cljindex 01758de2184..92ca02ad8e4 100644--- a/src/metabase/transforms_rest/api/transform_job.clj+++ b/src/metabase/transforms_rest/api/transform_job.clj@@ -5,8 +5,9 @@[metabase.api.macros :as api.macros][metabase.api.routes.common :refer [+auth]][metabase.models.interface :as mi]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.core :as transforms.core]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms.util :as transforms.u][metabase.util.i18n :refer [deferred-tru LocalizedString]][metabase.util.jvm :as u.jvm][metabase.util.log :as log]@@ -203,7 +204,7 @@(defn- add-next-run[{id :id :as job}](if-let [start-time (-> id transforms.core/existing-trigger :next-fire-time)]- (assoc job :next_run {:start_time (str (transforms.util/->instant start-time))})+ (assoc job :next_run {:start_time (str (transforms-base.u/->instant start-time))})job))(api.macros/defendpoint :get "/:job-id/transforms" :- [:sequential TransformResponse]@@ -215,7 +216,7 @@(-> (transforms.core/job-transforms job-id)(#(do (api/check-403 (every? mi/can-read? %)) %))(t2/hydrate :creator)- transforms.util/add-source-readable))+ transforms.u/add-source-readable))(api.macros/defendpoint :get "/" :- [:sequential TransformJobResponse]"Get all transform jobs."@@ -231,12 +232,12 @@(let [jobs (t2/select :model/TransformJob {:order-by [[:created_at :desc]]})](into [](comp (map add-next-run)- (transforms.util/->date-field-filter-xf [:last_run :start_time] last-run-start-time)- (transforms.util/->date-field-filter-xf [:next_run :start_time] next-run-start-time)- (transforms.util/->status-filter-xf [:last_run :status] last-run-statuses)- (transforms.util/->tag-filter-xf [:tag_ids] tag-ids)- (map #(update % :last_run transforms.util/localize-run-timestamps))- (map #(update % :next_run transforms.util/localize-run-timestamps)))+ (transforms-base.u/->date-field-filter-xf [:last_run :start_time] last-run-start-time)+ (transforms-base.u/->date-field-filter-xf [:next_run :start_time] next-run-start-time)+ (transforms-base.u/->status-filter-xf [:last_run :status] last-run-statuses)+ (transforms-base.u/->tag-filter-xf [:tag_ids] tag-ids)+ (map #(update % :last_run transforms-base.u/localize-run-timestamps))+ (map #(update % :next_run transforms-base.u/localize-run-timestamps)))(t2/hydrate jobs :tag_ids :last_run))))(def ^{:arglists '([request respond raise])} routesdiff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.cljindex dd86df90910..7026ab93ed0 100644--- a/test/metabase/transforms/execute_test.clj+++ b/test/metabase/transforms/execute_test.clj@@ -11,11 +11,11 @@[metabase.test :as mt][metabase.test.data.interface :as tx][metabase.test.data.sql :as sql.tx]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms.execute :as transforms.execute][metabase.transforms.query-test-util :as query-test-util][metabase.transforms.test-dataset :as transforms-dataset][metabase.transforms.test-util :as transforms.tu :refer [delete-schema! with-transform-cleanup!]]- [metabase.transforms.util :as transforms.util][metabase.util :as u][toucan2.core :as t2]))@@ -140,7 +140,7 @@{:name "name" :type :type/Text :nullable? true}]}](mt/as-admin(testing "create-table-from-schema! should create the table successfully"- (transforms.util/create-table-from-schema! driver db-id table-schema)+ (transforms-base.u/create-table-from-schema! driver db-id table-schema)(let [table-exists? (driver/table-exists? driver db-id {:schema schema-name :name table-name})](is (some? table-exists?) "Table should exist in the database schema")(driver/drop-table! driver db-id (:name table-schema))))))))diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.cljindex c96ae368705..a13f664e4c6 100644--- a/test/metabase/transforms/jobs_test.clj+++ b/test/metabase/transforms/jobs_test.clj@@ -152,12 +152,12 @@run-id 102logged-messages (atom [])run-called? (atom false)]- (with-redefs [log/log* (fn [_ level _ message]- (swap! logged-messages conj {:level level :message message}))- transform-run/running-run-for-transform-id (constantly nil)- transforms.execute/execute! (fn [_ _]- (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]+ (swap! logged-messages conj {:level level :message message}))+ transform-run/running-run-for-transform-id (constantly nil)+ transforms.execute/execute! (fn [_ _]+ (reset! run-called? true))+ transforms.job-run/add-run-activity! (constantly nil)](#'jobs/run-transform! run-id :scheduled nil query-transform)(is (empty? (filter (comp #{:warn} :level) @logged-messages))"Should not log warnings when feature is enabled")@@ -170,9 +170,9 @@:name "Test Query Transform"}run-id 100logged-messages (atom [])]- (with-redefs [log/log* (fn [_ level _ message]- (swap! logged-messages conj {:level level :message message}))- transform-run/running-run-for-transform-id (constantly nil)]+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]+ (swap! logged-messages conj {:level level :message message}))+ transform-run/running-run-for-transform-id (constantly nil)](#'jobs/run-transform! run-id :scheduled nil query-transform)(is (= 1 (count @logged-messages))"Should log exactly one warning")@@ -190,12 +190,12 @@run-id 102logged-messages (atom [])run-called? (atom false)]- (with-redefs [log/log* (fn [_ level _ message]- (swap! logged-messages conj {:level level :message message}))- transform-run/running-run-for-transform-id (constantly nil)- transforms.execute/execute! (fn [_ _]- (reset! run-called? true))- transforms.job-run/add-run-activity! (constantly nil)]+ (mt/with-dynamic-fn-redefs [log/log* (fn [_ level _ message]+ (swap! logged-messages conj {:level level :message message}))+ transform-run/running-run-for-transform-id (constantly nil)+ transforms.execute/execute! (fn [_ _]+ (reset! run-called? true))+ transforms.job-run/add-run-activity! (constantly nil)](#'jobs/run-transform! run-id :scheduled nil query-transform)(is (empty? (filter (comp #{:warn} :level) @logged-messages))"Should not log warnings when feature is enabled")@@ -237,9 +237,9 @@:tag_id (:id tag):position 0}](let [run-id-atom (atom nil)]- (with-redefs [jobs/run-transforms! (fn [run-id & _]- (reset! run-id-atom run-id)- (throw (ex-info "Uncaught error" {})))]+ (mt/with-dynamic-fn-redefs [jobs/run-transforms! (fn [run-id & _]+ (reset! run-id-atom run-id)+ (throw (ex-info "Uncaught error" {})))](try(jobs/run-job! (:id job) {:run-method :cron})(catch clojure.lang.ExceptionInfo _))@@ -286,9 +286,9 @@:tag_id (:id tag):position 0}](let [run-id-atom (atom nil)]- (with-redefs [jobs/run-transforms! (fn [run-id & _]- (reset! run-id-atom run-id)- (throw (ex-info "Uncaught error" {})))]+ (mt/with-dynamic-fn-redefs [jobs/run-transforms! (fn [run-id & _]+ (reset! run-id-atom run-id)+ (throw (ex-info "Uncaught error" {})))](try(jobs/run-job! (:id job) {:run-method :manual})(catch clojure.lang.ExceptionInfo _))@@ -481,14 +481,14 @@on-enter (fn [] (await-barrier barrier-enter))on-exit (fn [] (await-barrier barrier-exit) (.set tl-tripped true))original-insert transforms.u/try-start-unless-already-running]- (with-redefs [transforms.u/try-start-unless-already-running- (fn [transform-id run-method user-id]- (on-enter)- (let [[ret ex] (try- [(original-insert transform-id run-method user-id)]- (catch Throwable t [nil t]))]- (on-exit)- (if ex (throw ex) ret)))]+ (mt/with-dynamic-fn-redefs [transforms.u/try-start-unless-already-running+ (fn [transform-id run-method user-id]+ (on-enter)+ (let [[ret ex] (try+ [(original-insert transform-id run-method user-id)]+ (catch Throwable t [nil t]))]+ (on-exit)+ (if ex (throw ex) ret)))](let [run1 (transforms.job-run/start-run! (:id job) :manual)run2 (transforms.job-run/start-run! (:id job) :manual)fut1 (futurediff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex a86444bb11a..b307976b702 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -1,17 +1,20 @@(ns ^:mb/driver-tests metabase.transforms.util-test"Tests for transform utility functions."(:require+ [clojure.core.async :as a][clojure.test :refer :all][metabase.api.common :as api][metabase.driver :as driver][metabase.driver.util :as driver.u]- [metabase.events.core :as events][metabase.lib.core :as lib][metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions-group :as perms-group][metabase.test :as mt][metabase.test.data.sql :as sql.tx]- [metabase.transforms.util :as transforms.util]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.execute :as transforms.execute]+ [metabase.transforms.util :as transforms.u][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -64,8 +67,8 @@(testing "tables with schema"(let [table-with-schema {:name (name (driver.u/temp-table-name :postgres :schema/orders))}table-without-schema {:name (name (driver.u/temp-table-name :postgres :orders))}]- (is (true? (transforms.util/is-temp-transform-table? table-with-schema)))- (is (true? (transforms.util/is-temp-transform-table? table-without-schema)))))))+ (is (true? (transforms.u/is-temp-transform-table? table-with-schema)))+ (is (true? (transforms.u/is-temp-transform-table? table-without-schema)))))))(deftest create-table-from-schema!-test(testing "create-table-from-schema! preserves column order from schema definition"@@ -85,7 +88,7 @@(mt/as-admin(try(testing "Creating table with ordered columns"- (transforms.util/create-table-from-schema! driver db-id table-schema)+ (transforms-base.u/create-table-from-schema! driver db-id table-schema)(is (driver/table-exists? driver (mt/db) {:schema schema-name :name (name table-name)})))(when (get-method driver/describe-table driver)@@ -112,7 +115,7 @@(deftest ^:parallel matching-timestamp?-test(testing "matching-timestamp? checks if a timestamp falls within a date range [start, end)"- (let [matching-timestamp? #'transforms.util/matching-timestamp?+ (let [matching-timestamp? #'transforms-base.u/matching-timestamp?field-path [:start_time]range-jan-feb {:start "2024-01-01T00:00:00Z" :end "2024-02-01T00:00:00Z"}range-start-only {:start "2024-01-01T00:00:00Z" :end nil}@@ -155,30 +158,30 @@:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}:model/Table t2 {:db_id (:id db) :name "table_two" :schema "my_schema"}](testing "returns nil for empty input"- (is (nil? (transforms.util/batch-lookup-table-ids [])))- (is (nil? (transforms.util/batch-lookup-table-ids nil))))+ (is (nil? (transforms-base.u/batch-lookup-table-ids [])))+ (is (nil? (transforms-base.u/batch-lookup-table-ids nil))))(testing "looks up table without schema"(let [refs [{:database_id (:id db) :schema nil :table "table_one"}]- result (transforms.util/batch-lookup-table-ids refs)]+ result (transforms-base.u/batch-lookup-table-ids refs)](is (= {[(:id db) nil "table_one"] (:id t1)} result))))(testing "looks up table with schema"(let [refs [{:database_id (:id db) :schema "my_schema" :table "table_two"}]- result (transforms.util/batch-lookup-table-ids refs)]+ result (transforms-base.u/batch-lookup-table-ids refs)](is (= {[(:id db) "my_schema" "table_two"] (:id t2)} result))))(testing "handles mixed refs with and without schema"(let [refs [{:database_id (:id db) :schema nil :table "table_one"}{:database_id (:id db) :schema "my_schema" :table "table_two"}]- result (transforms.util/batch-lookup-table-ids refs)]+ result (transforms-base.u/batch-lookup-table-ids refs)](is (= {[(:id db) nil "table_one"] (:id t1)[(:id db) "my_schema" "table_two"] (:id t2)}result))))(testing "returns empty for non-existent table"(let [refs [{:database_id (:id db) :schema nil :table "nonexistent"}]- result (transforms.util/batch-lookup-table-ids refs)]+ result (transforms-base.u/batch-lookup-table-ids refs)](is (= {} result)))))))(deftest normalize-source-tables-test@@ -186,7 +189,7 @@(mt/with-temp [:model/Database db {}:model/Table t1 {:db_id (:id db) :name "existing_table" :schema nil}](testing "converts integer table ID to map format"- (let [result (transforms.util/normalize-source-tables {"t" (:id t1)})]+ (let [result (transforms-base.u/normalize-source-tables {"t" (:id t1)})](is (map? (get result "t")))(is (= (:id db) (get-in result ["t" :database_id])))(is (= "existing_table" (get-in result ["t" :table])))@@ -194,27 +197,27 @@(testing "throws for non-existent integer table ID"(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found for ids: 999999"- (transforms.util/normalize-source-tables {"t" 999999}))))+ (transforms-base.u/normalize-source-tables {"t" 999999}))))(testing "populates table_id for existing table"(let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table"}}- result (transforms.util/normalize-source-tables source-tables)]+ result (transforms-base.u/normalize-source-tables source-tables)](is (= (:id t1) (get-in result ["t" :table_id])))))(testing "preserves existing table_id"(let [source-tables {"t" {:database_id (:id db) :schema nil :table "existing_table" :table_id 999}}- result (transforms.util/normalize-source-tables source-tables)]+ result (transforms-base.u/normalize-source-tables source-tables)](is (= 999 (get-in result ["t" :table_id])))))(testing "leaves table_id nil for non-existent table ref"(let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}- result (transforms.util/normalize-source-tables source-tables)]+ result (transforms-base.u/normalize-source-tables source-tables)](is (nil? (get-in result ["t" :table_id])))))(testing "handles mixed int and map entries"(let [source-tables {"t1" (:id t1)"t2" {:database_id (:id db) :schema nil :table "existing_table"}}- result (transforms.util/normalize-source-tables source-tables)]+ result (transforms-base.u/normalize-source-tables source-tables)](is (map? (get result "t1")))(is (= (:id t1) (get-in result ["t1" :table_id])))(is (= (:id t1) (get-in result ["t2" :table_id]))))))))@@ -225,31 +228,31 @@:model/Table t1 {:db_id (:id db) :name "table_one" :schema nil}:model/Table t2 {:db_id (:id db) :name "table_two" :schema nil}](testing "passes through integer entries (old format)"- (is (= {"t" 123} (transforms.util/resolve-source-tables {"t" 123}))))+ (is (= {"t" 123} (transforms-base.u/resolve-source-tables {"t" 123}))))(testing "resolves map with table_id"(let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one" :table_id (:id t1)}}]- (is (= {"t" (:id t1)} (transforms.util/resolve-source-tables source-tables)))))+ (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))(testing "looks up table_id for map without it"(let [source-tables {"t" {:database_id (:id db) :schema nil :table "table_one"}}]- (is (= {"t" (:id t1)} (transforms.util/resolve-source-tables source-tables)))))+ (is (= {"t" (:id t1)} (transforms-base.u/resolve-source-tables source-tables)))))(testing "throws for non-existent table"(let [source-tables {"t" {:database_id (:id db) :schema nil :table "nonexistent"}}](is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: nonexistent"- (transforms.util/resolve-source-tables source-tables)))))+ (transforms-base.u/resolve-source-tables source-tables)))))(testing "throws with schema in error message"(let [source-tables {"t" {:database_id (:id db) :schema "my_schema" :table "nonexistent"}}](is (thrown-with-msg? clojure.lang.ExceptionInfo #"Tables not found: my_schema\.nonexistent"- (transforms.util/resolve-source-tables source-tables)))))+ (transforms-base.u/resolve-source-tables source-tables)))))(testing "handles mixed entries (old and new format)"(let [source-tables {"t1" (:id t1)"t2" {:database_id (:id db) :schema nil :table "table_two"}}](is (= {"t1" (:id t1) "t2" (:id t2)}- (transforms.util/resolve-source-tables source-tables))))))))+ (transforms-base.u/resolve-source-tables source-tables))))))))(deftest source-tables-readable?-test(testing "source-tables-readable? function"@@ -262,17 +265,17 @@(testing "returns true for query transform when user can read database"(let [transform {:source {:type :query:query {:database db-id}}}]- (is (true? (transforms.util/source-tables-readable? transform)))))+ (is (true? (transforms.u/source-tables-readable? transform)))))(testing "returns true for python transform when user can read all source tables"(let [transform {:source {:type :python:source-tables {"t1" table-id}}}]- (is (true? (transforms.util/source-tables-readable? transform)))))+ (is (true? (transforms.u/source-tables-readable? transform)))))(testing "handles source tables with table_id in map format"(let [transform {:source {:type :python:source-tables {"t1" {:table_id table-id}}}}]- (is (true? (transforms.util/source-tables-readable? transform))))))))))))+ (is (true? (transforms.u/source-tables-readable? transform))))))))))))(deftest source-tables-readable-permissions-test(testing "source-tables-readable? with various permission levels"@@ -290,7 +293,7 @@(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked(mt/with-db-perm-for-group! group db-id :perms/view-data :blocked(binding [api/*current-user-id* (:id user)]- (is (false? (transforms.util/source-tables-readable? transform))+ (is (false? (transforms.u/source-tables-readable? transform))"User with blocked database access should not be able to read source database")))))))(testing "Python transforms - blocked database access"@@ -301,7 +304,7 @@(mt/with-db-perm-for-group! (perms-group/all-users) db-id :perms/view-data :blocked(mt/with-db-perm-for-group! group db-id :perms/view-data :blocked(binding [api/*current-user-id* (:id user)]- (is (false? (transforms.util/source-tables-readable? transform))+ (is (false? (transforms.u/source-tables-readable? transform))"User with blocked database access should not be able to read source tables")))))))(testing "Python transforms - granular access but missing some tables"@@ -317,25 +320,31 @@;; Block table2 for the test group only(data-perms/set-table-permission! (:id group) table2-id :perms/view-data :blocked)(binding [api/*current-user-id* (:id user)]- (is (false? (transforms.util/source-tables-readable? transform))+ (is (false? (transforms.u/source-tables-readable? transform))"User who cannot read all source tables should have source_readable=false")))))))))))))-(deftest handle-transform-complete-sets-transform-id-test- (testing "handle-transform-complete! sets transform_id on the target table"- (mt/with-premium-features #{:transforms}- (let [target {:type "table" :schema nil :name "test_output_table"}]- (mt/with-temp [:model/Database {db-id :id :as db} {:engine :h2}- :model/Transform {transform-id :id :as transform} {:target target}- :model/Table {table-id :id :as table} {:db_id db-id :name "test_output_table" :schema nil}- :model/TransformRun {run-id :id} {:transform_id transform-id- :status "running"- :run_method "manual"}]- (with-redefs [transforms.util/sync-target! (constantly table)- events/publish-event! (constantly nil)- transforms.util/execute-secondary-index-ddl-if-required! (constantly nil)]- (transforms.util/handle-transform-complete! :run-id run-id :transform transform :db db)- (is (= transform-id- (t2/select-one-fn :transform_id :model/Table :id table-id)))))))))+(deftest execute-sets-transform-id-on-target-table-test+ (testing "Executing a query transform sets transform_id on the target table"+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/with-premium-features #{:transforms}+ (let [target {:type "table" :schema nil :name "test_output_table"}]+ (mt/with-temp [:model/Transform {transform-id :id :as transform}+ {:target target+ :source {:type "query"+ :query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}+ :model/Table {table-id :id} {:db_id (mt/id) :name "test_output_table" :schema nil}]+ ;; Mock execute-base! to return success without actually running a query,+ ;; run-cancelable-transform! to bypass schema creation / cancellation infra,+ ;; and sync/indexes to skip driver calls. complete-execution! runs normally+ ;; so it sets transform_id on the target table.+ (with-redefs [transforms-base.i/execute-base! (constantly {:status :succeeded})+ transforms-base.u/sync-target! (constantly nil)+ transforms-base.u/execute-secondary-index-ddl-if-required! (constantly nil)+ transforms.u/run-cancelable-transform! (fn [_run-id _driver _details run-fn & _opts]+ (run-fn (a/promise-chan)))]+ (transforms.execute/execute! transform {:run-method :manual})+ (is (= transform-id+ (t2/select-one-fn :transform_id :model/Table :id table-id))))))))))(deftest transform-hydration-test(testing "hydrating :transform on a table"@@ -356,7 +365,7 @@(deftest ^:parallel massage-sql-query-test(testing "massage-sql-query sets disable-remaps? and disable-max-results?"(let [query {:database 1, :type :query, :query {:source-table 1}}- massaged (transforms.util/massage-sql-query query)]+ massaged (transforms-base.u/massage-sql-query query)](is (true? (get-in massaged [:middleware :disable-remaps?])))(is (true? (get-in massaged [:middleware :disable-max-results?]))))))@@ -366,7 +375,7 @@(mt/with-premium-features #{:transforms}(let [transform {:source {:type "query":query (lib/query (mt/metadata-provider) (mt/mbql-query venues))}}- {:keys [query]} (transforms.util/compile-source transform)]+ {:keys [query]} (transforms-base.u/compile-source transform)](is (string? query))(is (not (re-find #"(?i)\bLIMIT\b" query))(str "Expected no LIMIT clause in compiled SQL, got: " query)))))))diff --git a/test/metabase/transforms/ordering_test.clj b/test/metabase/transforms_base/ordering_test.cljsimilarity index 98%rename from test/metabase/transforms/ordering_test.cljrename to test/metabase/transforms_base/ordering_test.cljindex 6cc78a2910f..c390b54bfcf 100644--- a/test/metabase/transforms/ordering_test.clj+++ b/test/metabase/transforms_base/ordering_test.clj@@ -1,4 +1,4 @@-(ns ^:mb/driver-tests metabase.transforms.ordering-test+(ns ^:mb/driver-tests metabase.transforms-base.ordering-test(:require[clojure.test :refer :all][metabase.driver :as driver]@@ -6,9 +6,9 @@[metabase.query-processor.compile :as qp.compile][metabase.test :as mt][metabase.test.data.sql :as sql.tx]+ [metabase.transforms-base.interface :as transforms-base.i]+ [metabase.transforms-base.ordering :as ordering][metabase.transforms.execute :as transforms.execute]- [metabase.transforms.interface :as transforms.i]- [metabase.transforms.ordering :as ordering][toucan2.core :as t2]))(defn- default-schema-or-public [& [fallback-driver]]@@ -58,7 +58,7 @@(defn- transform-deps-for-db [transform](mt/with-metadata-provider (mt/id)- (#'transforms.i/table-dependencies transform)))+ (transforms-base.i/table-dependencies transform)))(deftest not-run-transform-dependency-ordering-test(mt/test-driver (mt/normal-driver-select {:+parent :sql-jdbc})diff --git a/test/metabase/transforms_inspector/lens_output_test.clj b/test/metabase/transforms_inspector/lens_output_test.cljindex 41453dba6d1..0cc4de3818b 100644--- a/test/metabase/transforms_inspector/lens_output_test.clj+++ b/test/metabase/transforms_inspector/lens_output_test.clj@@ -6,9 +6,9 @@[metabase.lib.metadata :as lib.metadata][metabase.query-processor.preprocess :as qp.preprocess][metabase.test :as mt]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms-inspector.core :as inspector.core][metabase.transforms-inspector.lens.core :as lens.core]- [metabase.transforms.util :as transforms.util][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -52,7 +52,7 @@(lib/with-join-alias "Products")))])(lib/with-join-alias "Products")(lib/with-join-fields :all)))- transforms.util/massage-sql-query+ transforms-base.u/massage-sql-queryqp.preprocess/preprocess)))(defn- extract-join-structure@@ -100,7 +100,7 @@(let [orders-info (build-table-info (mt/id :orders))mp (mt/metadata-provider)preprocessed (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))- transforms.util/massage-sql-query+ transforms-base.u/massage-sql-queryqp.preprocess/preprocess)]{:transform {}:source-type :mbqldiff --git a/test/metabase/transforms_rest/api/transform_test.clj b/test/metabase/transforms_rest/api/transform_test.cljindex 3d35e9aa173..08a232bb38d 100644--- a/test/metabase/transforms_rest/api/transform_test.clj+++ b/test/metabase/transforms_rest/api/transform_test.clj@@ -12,6 +12,7 @@[metabase.query-processor :as qp][metabase.search.test-util :as search.tu][metabase.test :as mt]+ [metabase.transforms-base.util :as transforms-base.u][metabase.transforms-rest.api.transform][metabase.transforms.crud][metabase.transforms.query-test-util :as query-test-util]@@ -20,7 +21,6 @@parse-instantutc-timestampwith-transform-cleanup!]]- [metabase.transforms.util :as transforms.util][metabase.util :as u][toucan2.core :as t2]))@@ -647,7 +647,7 @@(m/dissoc-in [:source :query :lib/metadata]))(-> (mt/user-http-request :lucky :put 200 (format "transform/%s" (:id resp)) updated)(update-in [:source :query] lib/normalize))))- (is (false? (transforms.util/target-table-exists? original)))))))))))+ (is (false? (transforms-base.u/target-table-exists? original)))))))))))(deftest delete-transforms-test(mt/with-premium-features #{}@@ -776,7 +776,7 @@original)_ (do (test-run! transform-id)(wait-for-table table1-name 5000))- _ (is (true? (transforms.util/target-table-exists? original)))+ _ (is (true? (transforms-base.u/target-table-exists? original)))_ (check-query-results table1-name [5 11 16] "Gadget")updated {:name "Doohickey Products":description "Desc"@@ -790,8 +790,8 @@(update-in [:source :query] lib/normalize))))(test-run! transform-id)(wait-for-table table2-name 5000)- (is (true? (transforms.util/target-table-exists? original)))- (is (true? (transforms.util/target-table-exists? updated)))+ (is (true? (transforms-base.u/target-table-exists? original)))+ (is (true? (transforms-base.u/target-table-exists? updated)))(check-query-results table2-name [2 3 4 13] "Doohickey")))))))))))(deftest get-runs-filter-by-single-transform-id-test