The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
(ns metabase-enterprise.transforms-python.base
"Base Python transform execution - core logic without transform_run tracking.
This namespace handles Python transform execution and returns results in memory
rather than writing to transform_run rows. Logs are returned in the result map
instead of being saved to transform_run.message."
(:require
[clojure.core.async :as a]
[clojure.java.io :as io]
[clojure.string :as str]
[metabase-enterprise.transforms-python.python-runner :as python-runner]
[metabase-enterprise.transforms-python.s3 :as s3]
[metabase-enterprise.transforms-python.settings :as transforms-python.settings]
[metabase.driver :as driver]
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.schema :as transforms-base.schema]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.util :as u]
[metabase.util.format :as u.format]
[metabase.util.i18n :as i18n]
[metabase.util.log :as log]
[metabase.util.malli :as mu])
(:import
(java.io File)
(java.nio.file Files)
(java.nio.file.attribute FileAttribute)))
(set! *warn-on-reflection* true)
;;; ------------------------------------------------- Interface Implementations -------------------------------------------------
(defmethod transforms-base.i/target-db-id :python
[transform]
(-> transform :target :database))
(defmethod transforms-base.i/source-db-id :python
[transform]
(-> transform :source :source-database))
(defn- source-table-value->dependency
"Convert a source table value (int or ref map) to a dependency map."
[v]
(cond
;; Integer table ID - direct table dependency
(int? v)
{:table v}
;; Map with resolved table_id - direct table dependency
(:table_id v)
{:table (:table_id v)}
;; Map without table_id - return ref for ordering system to resolve
:else
{:table-ref (select-keys v [:database_id :schema :table])}))
(defmethod transforms-base.i/table-dependencies :python
[transform]
(into #{}
(map source-table-value->dependency)
(get-in transform [:source :source-tables])))
;;; ------------------------------------------------- Message Log (in-memory only) -------------------------------------------------
(defn empty-message-log
"Returns a new message log for accumulating execution logs in memory."
[]
(atom {:pre-python []
:python nil
:post-python []}))
(defn- log!
"Appends a string to the message log."
[message-log s]
(swap! message-log (fn [m] (update m (if (:python m) :post-python :pre-python) conj s)))
nil)
(defn- replace-python-logs!
"Called when events output is received to copy those events into the message-log."
[message-log events]
(swap! message-log assoc :python events)
nil)
(defn message-log->string
"Presents the content of the log as a string."
[message-log]
(let [{:keys [pre-python python post-python]} @message-log]
(str/join "\n" (concat pre-python
(map :message python)
post-python))))
;;; ------------------------------------------------- Transfer Strategies -------------------------------------------------
(defn- table-schema [table-name metadata indexes]
(cond-> {:name (if (keyword? table-name) table-name (keyword table-name))
:columns (mapv (fn [{:keys [name base_type]}]
{:name name
:type (python-runner/restricted-insert-type base_type)
:nullable? true})
(:fields metadata))}
;; Python builds the table via `create-table!` (not a CTAS), so this is where inline indexes get rendered.
(seq indexes) (assoc :indexes indexes)))
(defn- insert-data!
"Insert data from source into an existing table."
[driver db-id table-schema data-source]
(let [data-source (assoc data-source :table-schema table-schema)]
(driver/insert-from-source! driver db-id table-schema data-source)))
(defn- create-table-and-insert-data!
"Create a table from metadata and insert data from source."
[driver db-id table-schema data-source]
(transforms-base.u/create-table-from-schema! driver db-id table-schema)
(insert-data! driver db-id table-schema data-source))
(defn- transfer-with-rename-tables-strategy!
"Transfer data using the rename-tables*! multimethod with atomicity guarantees."
[driver db-id table-name metadata indexes data-source]
(let [source-table-name (transforms-base.u/temp-table-name driver (namespace table-name))
temp-table-name (u/poll {:thunk #(transforms-base.u/temp-table-name driver (namespace table-name))
:done? #(not= source-table-name %)
:interval-ms 1})]
(log/info "Using rename-tables strategy with atomicity guarantees")
(try
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata indexes) data-source)
(transforms-base.u/rename-tables! driver db-id {table-name temp-table-name
source-table-name table-name})
(transforms-base.u/drop-table! driver db-id temp-table-name)
(catch Exception e
(log/errorf "Failed to transfer data using rename-tables strategy: %s" (ex-message e))
(try
(transforms-base.u/drop-table! driver db-id source-table-name)
(catch Exception _))
(throw e)))))
(defn- transfer-with-create-drop-rename-strategy!
"Transfer data using create + drop + rename to minimize time without data."
[driver db-id table-name metadata indexes data-source]
(let [source-table-name (transforms-base.u/temp-table-name driver (namespace table-name))]
(log/info "Using create-drop-rename strategy to minimize downtime")
(try
(create-table-and-insert-data! driver db-id (table-schema source-table-name metadata indexes) data-source)
(transforms-base.u/drop-table! driver db-id table-name)
(driver/rename-table! driver db-id source-table-name table-name)
(catch Exception e
(log/errorf "Failed to transfer data using create-drop-rename strategy: %s" (ex-message e))
(try
(transforms-base.u/drop-table! driver db-id source-table-name)
(catch Exception _))
(throw e)))))
(defn- transfer-with-drop-create-fallback-strategy!
"Transfer data using drop + create fallback strategy."
[driver db-id table-name metadata indexes data-source]
(log/info "Using drop-create fallback strategy")
(try
(transforms-base.u/drop-table! driver db-id table-name)
(create-table-and-insert-data! driver db-id (table-schema table-name metadata indexes) data-source)
(catch Exception e
(log/errorf "Failed to transfer data using drop-create fallback strategy: %s" (ex-message e))
(throw e))))
(defn- upsert-with-merge-strategy!
"Upsert the staged data into `table-name`, keyed on `unique-key` (a vector of column names)."
[driver db table-name metadata data-source unique-key]
(let [db-id (:id db)
staging (transforms-base.u/temp-table-name driver (namespace table-name))]
(transforms-base.u/validate-merge-unique-key! unique-key (mapv :name (:fields metadata)))
(create-table-and-insert-data! driver db-id (table-schema staging metadata nil) data-source)
(try
(let [[sql & params] (sql.qp/format-honeysql driver {:select [:*], :from [staging]})
select {:query sql, :params (vec params)}
merge-spec {:unique-key unique-key
:columns (mapv :name (:fields metadata))}]
(driver/execute-raw-queries! driver (driver/connection-spec driver db)
(driver.sql/compile-merge driver table-name select merge-spec)))
(finally
(transforms-base.u/drop-table! driver db-id staging)))))
(defmulti ^:private transfer-file-to-db
{:arglists '([driver db transform metadata temp-file])}
(fn [_ _ transform _ _] (-> transform :target :type keyword)))
(defmethod transfer-file-to-db :table-incremental
[driver {db-id :id :as db}
{:keys [target] :as transform}
metadata temp-file]
;; Full incremental run (no watermark — first ever or after a checkpoint-config reset): behave
;; like non-incremental and drop-and-recreate rather than appending.
(if (transforms-base.u/full-incremental-run? transform)
((get-method transfer-file-to-db :table) driver db transform metadata temp-file)
;; Normal incremental: append (or merge) if table exists, create if it doesn't
(let [table-name (transforms-base.u/qualified-table-name driver target)
table-exists? (transforms-base.u/target-table-exists? transform)
data-source {:type :jsonl-file
:file temp-file}
unique-key (transforms-base.u/merge-target-unique-key transform)]
(cond
(not table-exists?)
(do
(log/info "New table")
(create-table-and-insert-data! driver db-id
(table-schema table-name metadata (:indexes target))
data-source))
unique-key
(upsert-with-merge-strategy! driver db table-name metadata data-source unique-key)
:else
;; Append into an existing table: no create, so no inline indexes to apply.
(insert-data! driver db-id (table-schema table-name metadata nil) data-source)))))
(defmethod transfer-file-to-db :table
[driver {db-id :id :as db}
{:keys [target] :as transform}
metadata temp-file]
(let [table-name (transforms-base.u/qualified-table-name driver target)
table-exists? (transforms-base.u/target-table-exists? transform)
indexes (:indexes target)
data-source {:type :jsonl-file
:file temp-file}]
(cond
(not table-exists?)
(do
(log/info "New table")
(create-table-and-insert-data! driver db-id (table-schema table-name metadata indexes) data-source))
(driver.u/supports? driver :atomic-renames db)
(transfer-with-rename-tables-strategy! driver db-id table-name metadata indexes data-source)
(driver.u/supports? driver :rename db)
(transfer-with-create-drop-rename-strategy! driver db-id table-name metadata indexes data-source)
:else
(transfer-with-drop-create-fallback-strategy! driver db-id table-name metadata indexes data-source))))
;;; ------------------------------------------------- Cancellation -------------------------------------------------
(defn- start-cancellation-process!
"Starts a core.async process that sends a cancellation request to the python executor if cancel-chan receives a value."
[server-url run-id cancel-chan]
(a/go (when (a/<! cancel-chan)
(python-runner/cancel-python-code-http-call! server-url run-id))))
;;; ------------------------------------------------- Core Execution -------------------------------------------------
(defn- count-jsonl-rows
"Non-blank line count of a JSONL temp file."
[^File file]
(with-open [rdr (io/reader file)]
(->> (line-seq rdr) (remove str/blank?) count)))
(mu/defn- run-python-transform-impl! :- ::driver/run-transform-result
"Core Python transform execution. Returns an open map carrying an integer `:rows-affected` (the JSONL row count),
plus the HTTP runner response's `:status`/`:body`. Conforms to [[::driver/run-transform-result]] so the Python
path is held to the same contract as the SQL `run-transform!` methods. Every non-success path throws.
Options:
- `with-stage-timing-fn` - optional, (fn [run-id stage thunk] result) for instrumentation"
[{:keys [source] :as transform} :- ::transforms-base.schema/transform
db :- ::lib.schema.metadata/database
run-id :- pos-int?
cancel-chan :- ::transforms-base.schema/chan
message-log :- ::transforms-base.schema/atom
{:keys [with-stage-timing-fn source-range-params]}
:- ::transforms-base.schema/execute-base-options]
;; Resolve name-based source table refs to table IDs (throws if any not found)
(let [resolved-source-tables (transforms-base.u/resolve-source-tables (:source-tables source))]
(with-open [shared-storage-ref (s3/open-shared-storage! resolved-source-tables)]
(let [driver (:engine db)
server-url (transforms-python.settings/python-runner-url)
_ (python-runner/copy-tables-to-s3! {:run-id run-id
:shared-storage @shared-storage-ref
:source (assoc source :source-tables resolved-source-tables)
:cancel-chan cancel-chan
:limit (:limit source)
:transform-id (:id transform)
:source-range-params source-range-params})
_ (start-cancellation-process! server-url run-id cancel-chan)
{:keys [status body] :as response}
(python-runner/execute-python-code-http-call!
{:server-url server-url
:code (:body source)
:run-id run-id
:source-tables resolved-source-tables
:shared-storage @shared-storage-ref})
output-manifest (python-runner/read-output-manifest @shared-storage-ref)
events (python-runner/read-events @shared-storage-ref)]
(replace-python-logs! message-log events)
(if (not= 200 status)
(throw (ex-info "Python runner call failed"
{:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))
:status-code 400
:api-status-code status
:body body
:events events
:timeout (:timeout body)}))
(try
(let [temp-path (Files/createTempFile "transform-output-" ".jsonl" (u/varargs FileAttribute))
temp-file ^File (.toFile temp-path)]
(when-not (seq (:fields output-manifest))
(throw (ex-info "No fields in metadata"
{:metadata output-manifest
:raw-body body
:events events})))
(try
(with-open [in (python-runner/open-output @shared-storage-ref)]
(io/copy in temp-file))
;; Transfer file to database with instrumentation
(let [file-size (.length temp-file)
rows-written (count-jsonl-rows temp-file)
do-transfer (fn [] (transfer-file-to-db driver db transform output-manifest temp-file))]
(if with-stage-timing-fn
(with-stage-timing-fn run-id [:import :file-to-dwh] do-transfer)
(do-transfer))
(transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size rows-written)
(assoc response :rows-affected rows-written))
(finally
(.delete temp-file))))
(catch Exception e
(log/errorf "Failed to create resulting table: %s" (ex-message e))
(throw (ex-info "Failed to create the resulting table"
{:transform-message (or (:transform-message (ex-data e))
(i18n/tru "Failed to create the resulting table"))}
e)))))))))
;;; ------------------------------------------------- Base Execution Entry Point -------------------------------------------------
(mu/defn run-python-transform! :- ::transforms-base.schema/execute-base-result
"Execute Python transform. Returns result map.
Does:
- Resolve source tables
- Copy to S3
- Call python runner
- Transfer result to DB
- Sync target
Does NOT:
- Create transform_run row
- Poll/save logs to transform_run.message
Options:
- `:cancelled?` - (fn [] boolean), polled to check for cancellation
- `:run-id` - optional, for cancellation signaling and instrumentation
- `:with-stage-timing-fn` - optional, (fn [run-id stage thunk] result)
- `:message-log` - optional, pre-created message log atom (for scheduled execution log polling)
- `:cancel-chan` - optional, pre-created promise-chan for cancellation (avoids bridging latency)
Returns:
{:status :succeeded | :failed | :cancelled | :timeout
:result <http response>
:logs <string>
:error <exception if failed>}"
[transform :- ::transforms-base.schema/transform
{:keys [cancelled? run-id with-stage-timing-fn message-log cancel-chan source-range-params]} :- [:maybe ::transforms-base.schema/execute-base-options]]
(assert (transforms-base.u/python-transform? transform) "Transform must be a python transform")
(let [message-log (or message-log (empty-message-log))]
(try
;; Check cancellation before starting
(when (and cancelled? (cancelled?))
(throw (ex-info "Transform cancelled before start" {:status :cancelled})))
(let [{:keys [target] transform-id :id} transform
db (driver.u/lib-database (:database target))
;; Use run-id if provided, otherwise generate a temp one for python runner
effective-run-id (or run-id (rand-int Integer/MAX_VALUE))
cancel-chan (or cancel-chan
(let [ch (a/promise-chan)]
;; Bridge cancelled? to cancel-chan
(when cancelled?
(a/go
(loop []
(when-not (a/poll! ch)
(if (cancelled?)
(a/>! ch :cancel!)
(do
(a/<! (a/timeout 100))
(recur)))))))
ch))
start-ms (u/start-timer)]
(log! message-log (i18n/tru "Executing Python transform"))
(log/info "Executing Python transform" transform-id)
(let [result (run-python-transform-impl! transform db effective-run-id cancel-chan message-log
{:with-stage-timing-fn with-stage-timing-fn
:source-range-params source-range-params})]
(log! message-log (i18n/tru "Python execution finished successfully in {0}"
(u.format/format-milliseconds (u/since-ms start-ms))))
;; Check cancellation after python
(when (and cancelled? (cancelled?))
(throw (ex-info "Transform cancelled after python execution" {:status :cancelled})))
{:status :succeeded
:result result
:logs (message-log->string message-log)
:source-range-params source-range-params}))
(catch Exception e
(let [data (ex-data e)
logs (message-log->string message-log)
error-message (or (:transform-message data) (ex-message e))]
(cond
(= :cancelled (:status data))
{:status :cancelled
:error e
:logs logs}
(:timeout data)
{:status :timeout
:error e
:logs (or (when-let [events (:events data)]
(str/join "\n" (map :message events)))
logs)}
:else
(do
;; the caller owns the failure (and logs it); this returns it in the result map
(log/debugf "Error executing Python transform: %s" (ex-message e))
{:status :failed
:error e
:logs (str logs "\n" error-message)})))))))
;;; ------------------------------------------------- Interface Implementation -------------------------------------------------
(defmethod transforms-base.i/execute-base! :python
[transform opts]
(run-python-transform! transform opts))