The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve python error feedback capability (#64053)
Dan Stonedd7845cc8331
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/util.clj b/enterprise/backend/src/metabase_enterprise/transforms/util.cljindex b091db4f67a..a05f599bae3 100644--- a/enterprise/backend/src/metabase_enterprise/transforms/util.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms/util.clj@@ -1,7 +1,6 @@(ns metabase-enterprise.transforms.util(:require[clojure.core.async :as a]- [clojure.string :as str][java-time.api :as t][metabase-enterprise.transforms.canceling :as canceling][metabase-enterprise.transforms.models.transform-run :as transform-run]@@ -64,8 +63,11 @@(throw e)))))(defn run-cancelable-transform!- "Execute a transform with cancellation support and proper error handling."- [run-id driver {:keys [db-id conn-spec output-schema]} run-transform!]+ "Execute a transform with cancellation support and proper error handling.++ Options:+ - `:ex-message-fn` change how caught exceptions are presented to the user in run logs, by default the same as clojure.core/ex-message"+ [run-id driver {:keys [db-id conn-spec output-schema]} run-transform! & {:keys [ex-message-fn] :or {ex-message-fn ex-message}}];; local run is responsible for status, using canceling lifecycle(try(when-not (driver/schema-exists? driver db-id output-schema)@@ -78,9 +80,7 @@(transform-run/succeed-started-run! run-id)ret)(catch Throwable t- (let [{:keys [transform-run-message]} (ex-data t)- message (str/join "\n" (remove str/blank? [transform-run-message (.getMessage t)]))]- (transform-run/fail-started-run! run-id {:message message}))+ (transform-run/fail-started-run! run-id {:message (ex-message-fn t)})(throw t))(finally(canceling/chan-end-run! run-id))))diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.cljindex 07a36320bf3..97ac30081f4 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj@@ -11,6 +11,8 @@[metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.util :as u]+ [metabase.util.format :as u.format]+ [metabase.util.i18n :as i18n][metabase.util.jvm :as u.jvm][metabase.util.log :as log][toucan2.core :as t2])@@ -41,7 +43,7 @@[](atom {:pre-python [] ; log! outputs previous to the python execution, i.e table read progress:python nil ; events json structured logs from the /logs endpoint- :post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target+ :post-python []})) ; log! outputs after the python execution, i.e. output reads, writes to target(defn- log!"Appends a string to the message log, the string is user facing and should be suitable for presentation as part of the `transform_run.message` field."@@ -265,19 +267,18 @@(replace-python-logs! message-log events))(if (not= 200 status)(throw (ex-info "Python runner call failed"- {:status-code 400- :api-status-code status- :body body- :events events- :transform-run-message (message-log->transform-run-message message-log)}))+ {:transform-message (i18n/tru "Python execution failure (exit code {0})" (:exit_code body "?"))+ :status-code 400+ :api-status-code status+ :body body+ :events events}))(try(let [temp-file (File/createTempFile "transform-output-" ".jsonl")](when-not (seq (:fields output-manifest))(throw (ex-info "No fields in metadata"{:metadata output-manifest:raw-body body- :events events- :transform-run-message (message-log->transform-run-message message-log)})))+ :events events})))(try(with-open [writer (io/writer temp-file)](.write writer ^String output))@@ -291,9 +292,19 @@(catch Exception e(log/error e "Failed to to create resulting table")(throw (ex-info "Failed to create the resulting table"- {:transform-run-message (message-log->transform-run-message message-log)}+ {:transform-message (or (:transform-message (ex-data e))+ ;; TODO keeping messaging the same at this level+ ;; should be more specific in underlying calls+ (i18n/tru "Failed to create the resulting table"))}e))))))))+(defn- exceptional-run-message [message-log ex]+ (str/join "\n" (remove str/blank? [(message-log->transform-run-message message-log)+ (or (:transform-message (ex-data ex))+ (if (instance? InterruptedException ex)+ (i18n/tru "Transform interrupted")+ (i18n/tru "Something went wrong")))])))+(defn execute-python-transform!"Execute a Python transform by calling the python runner.@@ -306,7 +317,7 @@{driver :engine :as db} (t2/select-one :model/Database (:database target)){run-id :id} (transforms.util/try-start-unless-already-running transform-id run-method)](some-> start-promise (deliver [:started run-id]))- (log! message-log "Executing Python transform")+ (log! message-log (i18n/tru "Executing Python transform"))(log/info "Executing Python transform" transform-id "with target" (pr-str target))(let [start-ms (u/start-timer)transform-details {:db-id (:id db)@@ -316,9 +327,10 @@:output-table (transforms.util/qualified-table-name driver target)}run-fn (fn [cancel-chan](run-python-transform! transform db run-id cancel-chan message-log)- (log! message-log (format "Python execution finished successfully in %s" (Duration/ofMillis (u/since-ms start-ms))))+ (log! message-log (i18n/tru "Python execution finished successfully in {0}" (u.format/format-milliseconds (u/since-ms start-ms))))(save-log-to-transform-run-message! run-id message-log))- result (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn)]+ ex-message-fn #(exceptional-run-message message-log %)+ result (transforms.util/run-cancelable-transform! run-id driver transform-details run-fn :ex-message-fn ex-message-fn)](transforms.instrumentation/with-stage-timing [run-id :table-sync](transforms.util/sync-target! target db run-id)){:run_id run-iddiff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.cljindex 60dc6d6e8dc..1adaf5599bd 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/python_runner.clj@@ -14,6 +14,7 @@;; TODO check that querying team are ok with us accessing this directly, otherwise make another plan^{:clj-kondo/ignore [:discouraged-namespace]}[metabase.query-processor.store :as qp.store]+ [metabase.util.i18n :as i18n][metabase.util.json :as json][metabase.util.log :as log][toucan2.core :as t2])@@ -246,7 +247,7 @@table-name->idcancel-chan]}];; TODO there's scope for some parallelism here, in particular across different databases- (doseq [table-id (vals table-name->id)+ (doseq [[table-name table-id] table-name->id:let [{:keys [s3-client bucket-name objects]} shared-storage{data-path :path} (get objects [:table table-id :data]){manifest-path :path} (get objects [:table table-id :manifest])]]@@ -278,6 +279,13 @@(s3/upload-file s3-client bucket-name manifest-path tmp-meta-file))(transforms.instrumentation/record-data-transfer! run-id :file-to-s3 (+ data-size meta-size) nil)))+ (catch InterruptedException ie (throw ie))+ (catch Throwable t+ (throw (ex-info "An error occurred while copying table data to S3"+ {:table-id table-id+ :transform-message (or (:transform-message (ex-data t))+ (i18n/tru "Failed to copy table contents to shared storage {0} ({1})" table-name table-id))}+ t)))(finally(safe-delete tmp-data-file)(safe-delete tmp-meta-file))))))