Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

8.8 KB
Raw Blame
(ns metabase-enterprise.transforms-python.execute
  (:require
   [clojure.core.async :as a]
   [clojure.string :as str]
   [metabase-enterprise.transforms-python.base :as base]
   [metabase-enterprise.transforms-python.db :as transforms-python.db]
   [metabase-enterprise.transforms-python.python-runner :as python-runner]
   [metabase.app-db.core :as app-db]
   [metabase.driver :as driver]
   [metabase.driver.connection :as driver.conn]
   [metabase.driver.util :as driver.u]
   [metabase.tracing.core :as tracing]
   [metabase.transforms-base.interface :as transforms-base.i]
   [metabase.transforms-base.util :as transforms-base.u]
   [metabase.transforms.instrumentation :as transforms.instrumentation]
   [metabase.transforms.util :as transforms.u]
   [metabase.util.i18n :as i18n]
   [metabase.util.jvm :as u.jvm]
   [metabase.util.log :as log])
  (:import
   (java.io Closeable)
   (java.net SocketException SocketTimeoutException)
   (java.time Duration)))

(set! *warn-on-reflection* true)

(defn- save-log-to-transform-run-message!
  "Saves the content of the log into the transform_run's message field. Lossy, string contains user facing information only."
  [run-id message-log]
  (transforms-python.db/update-run-message! run-id (base/message-log->string message-log)))

(def ^:private ^Duration python-message-loop-sleep-duration
  (Duration/ofMillis 1000))

(defn- python-message-update-loop!
  "Block while relevant log data is replicated from the runner into the message log.
  When new logs are received, the log data will be flushed to the transform_run.message field as a string.

  The loop will exit on error or when interrupted."
  [run-id message-log]
  (try
    (loop []
      (if (.isInterrupted (Thread/currentThread))
        (log/debug "Message update loop interrupted")
        (do (let [sleep-ms (.toMillis python-message-loop-sleep-duration)]
              (when (pos? sleep-ms) (Thread/sleep sleep-ms)))
            ;; a read timeout here is usually transient, so keep polling rather than ending the loop
            (if-let [{:keys [status body]} (try
                                             (python-runner/get-logs run-id)
                                             (catch SocketTimeoutException _
                                               (log/debugf "Timed out polling for logs, run-id: %s" run-id)
                                               nil))]
              (cond
                (<= 200 status 299)
                (let [{:keys [execution_id events]} body]
                  (if-not (= run-id execution_id)
                    (do (log/debugf "Run id did not match expected: %s actual: %s" run-id execution_id)
                        (recur))
                    (do
                      (swap! message-log assoc :python events)
                      (save-log-to-transform-run-message! run-id message-log)
                      (recur))))
                (= 404 status)
                (do
                  (log/debugf "No logs yet (or run finished), run-id: %s" run-id)
                  (recur))
                :else
                (do
                  (log/warnf "Unexpected status polling for logs %s, run-id: %s" status run-id)
                  (log/debug "Exiting due to poll error")))
              (recur)))))
    (catch SocketException se (when-not (= "Closed by interrupt" (ex-message se)) (throw se)))
    (catch InterruptedException _)
    (catch Throwable e
      (log/errorf "An exception was caught during msg update loop, run-id: %s: %s" run-id (ex-message e)))))

(defn- open-python-message-update-future! ^Closeable [run-id message-log]
  (if (app-db/in-transaction?)
    ;; in a transaction (such as under mt/with-temp), it is not safe to poll for logs
    (do
      (log/warnf "Log polling disabled as we are within a database transaction, run-id: %s" run-id)
      (reify Closeable (close [_])))
    ;; poll updates until execution is complete
    (let [cleanup (fn [fut]
                    (future-cancel fut)
                    (if (= ::timeout (try (deref fut 10000 ::timeout) (catch Throwable _)))
                      (log/fatalf "Log polling task did not respond to interrupt, run-id: %s" run-id)
                      (log/debugf "Log polling task done, run-id: %s" run-id)))
          fut     (u.jvm/in-virtual-thread*
                   (python-message-update-loop! run-id message-log))]
      (reify Closeable
        (close [_] (cleanup fut))))))

(defn- exceptional-run-message [message-log ex]
  (str/join "\n" (remove str/blank? [(base/message-log->string message-log)
                                     (or (:transform-message (ex-data ex))
                                         (if (instance? InterruptedException ex)
                                           (i18n/tru "Transform interrupted")
                                           (i18n/tru "Something went wrong")))])))

(defn execute-python-transform!
  "Execute a Python transform by calling the python runner.

  Blocks until the transform returns."
  [transform {:keys [run-method on-start user-id parent-run]}]
  (try
    (let [message-log                                                (base/empty-message-log)
          {:keys [target owner_user_id creator_id] transform-id :id} transform
          {driver :engine :as db}                                    (driver.u/lib-database (transforms-base.i/target-db-id transform))
          run-user-id                                                (if (and (= run-method :manual) user-id)
                                                                       user-id
                                                                       (or owner_user_id creator_id))
          {run-id :id}                                               (transforms.u/try-start-unless-already-running transform-id run-method run-user-id
                                                                                                                    :parent-run parent-run)]
      (when on-start (on-start run-id))
      (with-open [_ (open-python-message-update-future! run-id message-log)]
        (driver.conn/with-write-connection
          (let [target-type (keyword (:type target))]
            (tracing/with-span :tasks "task.transform.python"
              {:transform/id                   transform-id
               :transform/target-type          (name target-type)
               :transform/incremental          (transforms-base.u/incremental-target? transform)
               :transform/full-incremental-run (transforms-base.u/full-incremental-run? transform)
               :db/id                          (:id db)
               :db/engine                      (name driver)}
              (let [conn-spec         (driver/connection-spec driver db)
                    transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
                    run-fn            (fn [cancel-chan source-range-params]
                                        (let [result (transforms-base.i/execute-base!
                                                      transform
                                                      {:cancelled?           #(boolean (a/poll! cancel-chan))
                                                       :cancel-chan          cancel-chan
                                                       :run-id               run-id
                                                       :source-range-params  source-range-params
                                                       :message-log          message-log
                                                       :with-stage-timing-fn (fn [rid stage thunk]
                                                                               (transforms.instrumentation/with-stage-timing [rid stage]
                                                                                 (thunk)))})]
                                          (save-log-to-transform-run-message! run-id message-log)
                                          (when-not (= :succeeded (:status result))
                                            (throw (or (:error result) (ex-info "Transform failed" {:status (:status result)}))))
                                          result))
                    ex-message-fn     #(exceptional-run-message message-log %)
                    result            (transforms.instrumentation/with-stage-timing [run-id [:computation :python-execution]]
                                        (transforms.u/run-cancelable-transform! run-id transform driver transform-details run-fn :ex-message-fn ex-message-fn))]
                ;; Post-processing: sync, transform_id, events
                (transforms-base.u/complete-execution! transform {})
                {:run_id run-id :result result}))))))
    (catch Throwable t
      (log/errorf "Error executing Python transform %s: %s" (pr-str (:id transform)) (ex-message t))
      (throw t))))