Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Connection closed flake during python-transform-timeout-status-test round 2 (#64947)

Dan Stone2dc59090e34f
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index 9224cb7a77f..1ce5109613a 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -9,6 +9,7 @@
[metabase-enterprise.transforms.core :as transforms]
[metabase-enterprise.transforms.instrumentation :as transforms.instrumentation]
[metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.app-db.core :as app-db]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.util :as u]
@@ -241,7 +242,12 @@
(defn- run-python-transform! [{:keys [source] :as transform} db run-id cancel-chan message-log]
;; TODO restructure things such that s3 can we swapped out for other transfer mechanisms
- (with-open [log-future-ref (open-python-message-update-future! run-id message-log)
+ (with-open [^Closeable log-future-ref
+ (if (app-db/in-transaction?)
+ ;; if in a transaction (such as under mt/with-temp), it is not safe to poll for logs (close race / contention)
+ ;; tests that want to test async log behaviour should opt out of thread-local test helpers
+ (reify Closeable (close [_]))
+ (open-python-message-update-future! run-id message-log))
shared-storage-ref (s3/open-shared-storage! (:source-tables source))]
(let [driver (:engine db)
server-url (transforms-python.settings/python-runner-url)
@@ -261,7 +267,7 @@
output-manifest (python-runner/read-output-manifest @shared-storage-ref)
events (python-runner/read-events @shared-storage-ref)]
- (.close log-future-ref) ; early close to force any writes to flush
+ (.close ^Closeable log-future-ref) ; early close to force any writes to flush
(replace-python-logs! message-log events)
(if (not= 200 status)
(do
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
index 38a3bd9cfbb..b79f175b2d6 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/execute_test.clj
@@ -116,7 +116,6 @@
:body long-running-code}
:target (assoc target :database (mt/id))}]
(mt/with-temp [:model/Transform transform transform-def]
- (t2/query "SET LOCAL idle_in_transaction_session_timeout = 30000") ; mt/with-temp introduces a transaction which postgres might close early!
(let [{:keys [run_id]} (try
(transforms-python.execute/execute-python-transform! transform {:run-method :manual})
(catch Exception _