Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Use transform timeout for queries inside transforms (#72522)

Ngoc Khuate0502bb3207e
diff --git a/docs/configuring-metabase/environment-variables.md b/docs/configuring-metabase/environment-variables.md
index 39d337a87a0..ae914465a96 100644
--- a/docs/configuring-metabase/environment-variables.md
+++ b/docs/configuring-metabase/environment-variables.md
@@ -445,8 +445,7 @@ By default, this is 20 minutes.
Timeout in minutes for the database's query execution, both for the Metabase application database and any data connections.
If you have long-running queries, you might consider increasing this value. Adjusting the timeout does not impact Metabase’s frontend.
- This setting also applies to individual queries executed within transforms, so make sure the duration is long enough
- that it doesn't timeout any long-running queries in your transforms.
+ This setting does not apply to queries executed within transforms; those are governed by MB_TRANSFORM_TIMEOUT instead.
Please be aware that other services (like Nginx) may still drop long-running queries.
@@ -2197,8 +2196,9 @@ Timeout in milliseconds to wait after query cancellation before escalating to th
The timeout for a transform job, in minutes.
-Each query executed by a transform is also subject to the MB_DB_QUERY_TIMEOUT_MINUTES timeout,
- so make sure that value isn't lower, or it will timeout your transform.
+Controls the timeout for transform runs, including the queries they execute. This takes precedence
+ over MB_DB_QUERY_TIMEOUT_MINUTES for queries executed inside a transform, so transforms can run longer than regular
+ Metabase queries.
### `MB_TRANSFORMS_ENABLED`
diff --git a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj b/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
index a566cfb6f5c..a3397108494 100644
--- a/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
+++ b/modules/drivers/sparksql/src/metabase/driver/sparksql.clj
@@ -243,7 +243,11 @@
;; disabled for now, see issue #40991 to fix this.
:window-functions/cumulative false
:database-routing false
- :jdbc/statements false}]
+ :jdbc/statements false
+ ;; Calling Statement.setQueryTimeout on Hive's JDBC driver closes the Thrift transport
+ ;; on the server side; subsequent statement close() then throws "Failed to close
+ ;; statement". Skip the call entirely.
+ :jdbc/set-query-timeout false}]
(defmethod driver/database-supports? [:sparksql feature] [_driver _feature _db] supported?))
(defmethod sql.qp/quote-style :sparksql
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index cf2853130e7..0a25cd325dc 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -860,6 +860,11 @@
;; Does this driver support creating a java.sql.Statement via a Connection?
:jdbc/statements
+ ;; Can `Statement.setQueryTimeout` be called safely on this driver's statements? Defaults to true; set to
+ ;; false for drivers where calling it poisons the underlying session (e.g. SparkSQL, where the call closes the
+ ;; Thrift transport on the server side, causing subsequent statement close() to throw).
+ :jdbc/set-query-timeout
+
;; Does this driver provide :database-default on (describe-fields) or (describe-table)
:describe-default-expr
diff --git a/src/metabase/driver/connection.clj b/src/metabase/driver/connection.clj
index 82c1f8b49db..54088d1f72d 100644
--- a/src/metabase/driver/connection.clj
+++ b/src/metabase/driver/connection.clj
@@ -63,7 +63,7 @@
(def connection-types
"All valid values for [[*connection-type*]]."
- [:default :write-data :admin])
+ [:default :write-data :admin :transform])
(mr/def ::connection-type
(into [:enum] connection-types))
@@ -74,8 +74,13 @@
- `:default` — primary `:details`
- `:write-data` — `:write-data-details` merged over `:details` (if configured)
- `:admin` — `:admin-details` merged over `:details` (if configured)
+ - `:transform` — same details as `:write-data` (transforms write, so `:write-data-details`
+ are taken into account when configured), but resolved through a separate connection pool
+ whose c3p0 leak-detector tolerates transform-length runtimes. Set by
+ [[with-transform-connection]] from the transform runner.
- Bind via [[with-write-connection]], [[with-default-connection]], or [[with-admin-connection]], not directly."
+ Bind via [[with-write-connection]], [[with-default-connection]], [[with-admin-connection]], or
+ [[with-transform-connection]], not directly."
:default)
(defmacro with-write-connection
@@ -110,6 +115,17 @@
`(binding [*connection-type* :default]
~@body))
+(defmacro with-transform-connection
+ "Establishes a transform-connection context for body.
+
+ Connection details resolve the same as `:write-data` (transforms write, so
+ `:write-data-details` are taken into account when configured), but queries are routed
+ through a separate connection pool keyed on `:transform` so the pool can carry its own
+ c3p0 properties. See [[*connection-type*]] for the rationale."
+ [& body]
+ `(binding [*connection-type* :transform]
+ ~@body))
+
(def ^:dynamic ^:private *suppress-resolution-telemetry*
false)
@@ -145,13 +161,29 @@
(defn- overlay-details-for-type
"Returns the connection-type-specific details overlay for `database`, or nil if the
- requested type has no configured overlay (or is `:default`)."
+ requested type has no configured overlay (or is `:default`).
+
+ `:transform` resolves the same `:write-data-details` as `:write-data` — transforms write,
+ so they get write-data credentials when set. The two still resolve to *different* pool keys
+ (see [[effective-connection-type]]) so the pool properties can differ."
[database connection-type]
(case connection-type
:default nil
:write-data (database-write-data-details database)
+ :transform (database-write-data-details database)
:admin (database-admin-details database)))
+(defn- resolve-effective-type
+ "Maps a requested `connection-type` plus whether an `overlay` was resolved to the effective
+ pool key. `:transform` always keeps its own pool key — it is distinguished by its pool
+ properties, not its details — so it never falls back to `:default`. Other non-default types
+ fall back to `:default` when no overlay is configured, to avoid a duplicate pool."
+ [connection-type overlay]
+ (cond
+ (= connection-type :transform) :transform
+ overlay connection-type
+ :else :default))
+
(defn effective-details
"Returns the connection details map appropriate for the current context.
@@ -159,14 +191,18 @@
By default, returns the primary `:details`. Within a [[with-write-connection]] or
[[with-admin-connection]] scope, takes the corresponding `:write-data-details` /
- `:admin-details` into account (if configured). Within a
+ `:admin-details` into account (if configured). Within a [[with-transform-connection]]
+ scope, takes `:write-data-details` into account (if configured) — transforms write, so
+ they get write-data credentials when set — but resolves to a *different* pool key
+ (`:transform` vs `:write-data`) so the pool properties (e.g. `unreturnedConnectionTimeout`)
+ can differ. Within a
[[driver.w/with-swapped-connection-details]] scope, applies workspace isolation
overrides on top."
[database]
(when-let [database (some-> database driver.u/ensure-lib-database)]
(let [overlay (perf/not-empty (overlay-details-for-type database *connection-type*))
base (merge (:details database) overlay)
- eff-type (if overlay *connection-type* :default)]
+ eff-type (resolve-effective-type *connection-type* overlay)]
;; Track when an overlay is genuinely used (not fallback, not workspace-swapped).
;; Default resolutions are not tracked here — see :metabase-db-connection/write-op for
;; pool-level connection acquisition metrics.
@@ -191,6 +227,7 @@
(case connection-type
:default (:details database)
:write-data (database-write-data-details database)
+ :transform (:details database)
:admin (database-admin-details database))))
(defn write-connection-requested?
@@ -204,19 +241,20 @@
(= *connection-type* :admin))
(defn effective-connection-type
- "Returns the connection type actually in effect for the given database. Return value
- matches malli schema [[::connection-type]].
+ "Returns the effective pool key for the given database. Return value matches malli schema
+ [[::connection-type]].
Returns the requested non-default type only when both *requested* (via the
corresponding `with-*-connection`) and *configured* (the database has the relevant
overlay details). Otherwise returns `:default`, avoiding duplicate resource allocation
- (e.g. connection pools) when the requested type would resolve identically to `:default`."
+ (e.g. connection pools) when the requested type would resolve identically to `:default`.
+
+ `:transform` is the exception: it always resolves to `:transform` — the transform pool is
+ distinguished by its pool properties, not its details. See [[*connection-type*]]."
[database]
(let [database (driver.u/ensure-lib-database database)]
- (if (and (not= *connection-type* :default)
- (perf/not-empty (overlay-details-for-type database *connection-type*)))
- *connection-type*
- :default)))
+ (resolve-effective-type *connection-type*
+ (perf/not-empty (overlay-details-for-type database *connection-type*)))))
(defn track-connection-acquisition!
"Increments a Prometheus counter tracking connection acquisitions by connection type
diff --git a/src/metabase/driver/settings.clj b/src/metabase/driver/settings.clj
index 7ff38565a69..b1f4ad0fafe 100644
--- a/src/metabase/driver/settings.clj
+++ b/src/metabase/driver/settings.clj
@@ -98,8 +98,7 @@
:doc "Timeout in minutes for the database's query execution, both for the Metabase application database and any data connections.
If you have long-running queries, you might consider increasing this value. Adjusting the timeout does not impact Metabase’s frontend.
- This setting also applies to individual queries executed within transforms, so make sure the duration is long enough
- that it doesn't timeout any long-running queries in your transforms.
+ This setting does not apply to queries executed within transforms; those are governed by MB_TRANSFORM_TIMEOUT instead.
Please be aware that other services (like Nginx) may still drop long-running queries.")
@@ -151,9 +150,12 @@
(long (/ *query-timeout-ms* 1000))))
(defsetting jdbc-data-warehouse-unreturned-connection-timeout-seconds
- "Kill connections if they are unreturned after this amount of time. Currently, this is the mechanism that
- terminates JDBC driver queries that run too long. This should be the same as the query timeout in
- [[metabase.query-processor.context/query-timeout-ms]] and should not be overridden without a very good reason."
+ "Kill data-warehouse connections that have been checked out but not returned to the pool after this many seconds.
+ Acts as a leak-detector safety net — per-query timeouts are enforced separately via `Statement.setQueryTimeout`.
+ Defaults to the current `*query-timeout-ms*` in seconds, which is `MB_DB_QUERY_TIMEOUT_MINUTES` outside transforms
+ and `MB_TRANSFORM_TIMEOUT` inside [[metabase.driver.connection/with-transform-connection]] — so the transform pool
+ (a separate c3p0 pool keyed on `:transform`) gets a leak-detector tuned to transform-length runtimes without
+ weakening the leak-detector on the default pool used by ad-hoc queries."
:visibility :internal
:type :integer
:getter #'-jdbc-data-warehouse-unreturned-connection-timeout-seconds
diff --git a/src/metabase/driver/sql_jdbc.clj b/src/metabase/driver/sql_jdbc.clj
index 784c576ebaf..27369d5c956 100644
--- a/src/metabase/driver/sql_jdbc.clj
+++ b/src/metabase/driver/sql_jdbc.clj
@@ -75,6 +75,7 @@
(boolean (seq (sql-jdbc.execute/set-timezone-sql driver))))
(defmethod driver/database-supports? [:sql-jdbc :jdbc/statements] [_driver _feature _db] true)
+(defmethod driver/database-supports? [:sql-jdbc :jdbc/set-query-timeout] [_driver _feature _db] true)
(defmethod driver/db-default-timezone :sql-jdbc
[driver database]
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index e496883651b..1234d5d1d21 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -573,29 +573,47 @@
(log/debug "Query canceled, calling Statement.cancel()")
(.cancel stmt))))))
+(defn- set-statement-query-timeout!
+ "Set `Statement.setQueryTimeout` to the current `*query-timeout-ms*`. Applied uniformly to every SQL-JDBC statement
+ so each query carries its own server-side timeout, rather than relying on the pool-wide c3p0
+ `unreturnedConnectionTimeout` to kill long queries. Transforms rebind `*query-timeout-ms*` so their statements get
+ the transform timeout instead of the shorter default. Drivers that opt out via the `:jdbc/set-query-timeout`
+ feature flag (e.g. SparkSQL — calling it closes the Hive Thrift transport) skip the call entirely; for the rest,
+ individual implementations that throw fall back to the c3p0 leak-detector via the `catch Throwable`."
+ [driver ^Statement stmt]
+ (when (driver/database-supports? driver :jdbc/set-query-timeout nil)
+ (try
+ (.setQueryTimeout stmt (long (/ driver.settings/*query-timeout-ms* 1000)))
+ (catch Throwable e
+ (log/debug e "Error setting statement query timeout")))))
+
(defn- prepared-statement*
^PreparedStatement [driver conn sql params canceled-chan]
;; sometimes preparing the statement fails, usually if the SQL syntax is invalid. Match the
;; classification used in [[execute-reducible-query]] below: such errors should surface to the
;; user as :invalid-query (HTTP 4xx with the underlying database message), not :driver
;; (HTTP 5xx "We're experiencing server issues"). See #71637.
- (doto (try
- (prepared-statement driver conn sql params)
- (catch Throwable e
- (throw (ex-info (tru "Error preparing statement: {0}" (ex-message e))
- {:driver driver
- :type driver-api/qp.error-type.invalid-query
- :sql (str/split-lines (driver/prettify-native-form driver sql))
- :params params}
- e))))
- (wire-up-canceled-chan-to-cancel-Statement! canceled-chan)))
+ (let [stmt (try
+ (prepared-statement driver conn sql params)
+ (catch Throwable e
+ (throw (ex-info (tru "Error preparing statement: {0}" (ex-message e))
+ {:driver driver
+ :type driver-api/qp.error-type.invalid-query
+ :sql (str/split-lines (driver/prettify-native-form driver sql))
+ :params params}
+ e))))]
+ (set-statement-query-timeout! driver stmt)
+ (wire-up-canceled-chan-to-cancel-Statement! stmt canceled-chan)
+ stmt))
(defn- use-statement? [driver params]
(and (driver/database-supports? driver :jdbc/statements nil) (empty? params)))
(defn- statement* ^Statement [driver conn canceled-chan]
- (doto (statement driver conn)
- (wire-up-canceled-chan-to-cancel-Statement! canceled-chan)))
+ (let [stmt (statement driver conn)]
+ (set-statement-query-timeout! driver stmt)
+ (wire-up-canceled-chan-to-cancel-Statement! stmt canceled-chan)
+ stmt))
(defn statement-or-prepared-statement
"Create a statement or a prepared statement. Should be called from [[with-open]]."
diff --git a/src/metabase/transforms/settings.clj b/src/metabase/transforms/settings.clj
index e7cb0006d81..218e9d5e1e8 100644
--- a/src/metabase/transforms/settings.clj
+++ b/src/metabase/transforms/settings.clj
@@ -11,8 +11,11 @@
:type :integer
:visibility :internal
:default (* 4 60)
- :doc "Each query executed by a transform is also subject to the MB_DB_QUERY_TIMEOUT_MINUTES timeout,
- so make sure that value isn't lower, or it will timeout your transform."
+ :doc "Controls the timeout for transform runs, including the queries they execute. This takes precedence
+ over MB_DB_QUERY_TIMEOUT_MINUTES for queries executed inside a transform, so transforms can run longer than regular
+ Metabase queries. Enforced per-statement via `Statement.setQueryTimeout`; transforms also use a separate JDBC
+ connection pool whose c3p0 leak-detector tolerates this longer runtime, so non-transform connections continue to
+ use the shorter `MB_DB_QUERY_TIMEOUT_MINUTES` leak-detector."
:feature :transforms-basic
:export? false
:encryption :no
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index c11236ec722..ce0d0c447c8 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -9,6 +9,8 @@
[clojure.string :as str]
[metabase.api.common :as api]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
+ [metabase.driver.settings :as driver.settings]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.util :as driver.u]
[metabase.models.interface :as mi]
@@ -117,17 +119,27 @@
(try
(when-not (driver/schema-exists? driver db-id output-schema)
(driver/create-schema-if-needed! driver conn-spec output-schema))
- (let [source-range-params (transforms-base.u/get-source-range-params transform)]
+ (let [source-range-params (transforms-base.u/get-source-range-params transform)
+ transform-timeout (transforms.settings/transform-timeout)
+ transform-timeout-ms (u/minutes->ms transform-timeout)]
(transforms-base.u/save-run-checkpoint-range! run-id source-range-params)
;; Enrich the active task.transform.* span with checkpoint range attrs for
;; incremental runs. No-op when tracing is disabled or no span is active.
(when source-range-params
(tracing/add-span-attrs! :tasks (transforms-base.u/checkpoint-span-attrs source-range-params)))
- (canceling/chan-start-timeout-vthread! run-id (transforms.settings/transform-timeout))
+ (canceling/chan-start-timeout-vthread! run-id transform-timeout)
(let [cancel-chan (a/promise-chan)
- ret (binding [qp.pipeline/*canceled-chan* cancel-chan]
- (canceling/chan-start-run! run-id cancel-chan)
- (run-transform! cancel-chan source-range-params))]
+ ret (driver.conn/with-transform-connection
+ ;; Route through the `:transform` JDBC pool, whose `unreturnedConnectionTimeout` will be set
+ ;; from the `*query-timeout-ms*` binding below at pool-creation time. This keeps the default
+ ;; pool's leak-detector at `MB_DB_QUERY_TIMEOUT_MINUTES` for all non-transform traffic.
+ (binding [qp.pipeline/*canceled-chan* cancel-chan
+ driver.settings/*query-timeout-ms* transform-timeout-ms
+ ;; Match the query timeout so a single slow socket read (or a driver that waits for
+ ;; the full server-side query) does not get killed before the transform's own deadline.
+ driver.settings/*network-timeout-ms* (max driver.settings/*network-timeout-ms* transform-timeout-ms)]
+ (canceling/chan-start-run! run-id cancel-chan)
+ (run-transform! cancel-chan source-range-params)))]
(transforms-base.u/save-watermark! (:id transform) source-range-params)
(transform-run/succeed-started-run! run-id)
ret))
diff --git a/test/metabase/transforms/execute_test.clj b/test/metabase/transforms/execute_test.clj
index 98da3f1d2a3..0f501894d17 100644
--- a/test/metabase/transforms/execute_test.clj
+++ b/test/metabase/transforms/execute_test.clj
@@ -506,21 +506,21 @@
[(format "DROP OWNED BY %s;" readonly-user)]
[(format "DROP ROLE IF EXISTS %s;" readonly-user)]])))))))))
-(deftest transform-creates-write-pool-test
+(deftest transform-creates-transform-pool-test
(mt/test-drivers (mt/normal-driver-select {:+parent :sql-jdbc, :+features [:transforms/table]})
(when config/ee-available?
(mt/with-premium-features #{:writable-connection}
(mt/dataset transforms-dataset/transforms-test
- (let [db-id (mt/id)
- write-cache-key [db-id :write-data]
- schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
- old-write-details (:write_data_details (mt/db))]
+ (let [db-id (mt/id)
+ transform-cache-key [db-id :transform]
+ schema (t2/select-one-fn :schema :model/Table (mt/id :transforms_products))
+ old-write-details (:write_data_details (mt/db))]
(when-not old-write-details
(t2/update! :model/Database db-id {:write_data_details (:details (mt/db))}))
(try
(sql-jdbc.conn/invalidate-pool-for-db! (mt/db))
- (testing "write pool does not exist before transform execution"
- (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))
+ (testing "transform pool does not exist before transform execution"
+ (is (not (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool transform-cache-key))))
(with-transform-cleanup! [target-table {:type "table"
:schema schema
:name "pool_test"}]
@@ -535,8 +535,8 @@
:target target-table}]
(transforms.execute/execute! transform {:run-method :manual})
(transforms.tu/wait-for-table (:name target-table) 10000)
- (testing "write pool is created during transform execution"
- (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool write-cache-key))))))
+ (testing "transform pool is created during transform execution"
+ (is (contains? @@#'sql-jdbc.conn/pool-cache-key->connection-pool transform-cache-key))))))
(finally
(t2/update! :model/Database db-id {:write_data_details old-write-details})
(sql-jdbc.conn/invalidate-pool-for-db! (mt/db))))))))))
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index 8ece83794e9..390d7621450 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -6,6 +6,9 @@
[clojure.test :refer :all]
[metabase.api.common :as api]
[metabase.driver :as driver]
+ [metabase.driver.connection :as driver.conn]
+ [metabase.driver.settings :as driver.settings]
+ [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.permissions.models.data-permissions :as data-perms]
@@ -15,8 +18,11 @@
[metabase.test.data.sql :as sql.tx]
[metabase.transforms-base.interface :as transforms-base.i]
[metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.canceling :as transforms.canceling]
[metabase.transforms.execute :as transforms.execute]
+ [metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.util :as transforms.u]
+ [metabase.util :as u]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -408,6 +414,112 @@
(let [hydrated (t2/hydrate table :transform)]
(is (nil? (:transform hydrated))))))))
+(deftest transform-pool-uses-its-own-leak-detector-test
+ (testing "Inside a `with-transform-connection` scope, the data-warehouse pool's
+ `unreturnedConnectionTimeout` default tracks the transform's bound `*query-timeout-ms*` — not the
+ ambient `MB_DB_QUERY_TIMEOUT_MINUTES`. This is what makes transforms safe without weakening the leak
+ detector on the default pool: the `:transform` pool is created with a different value (transform-timeout)
+ than the `:default` pool (db-query-timeout)."
+ (mt/with-temporary-setting-values [jdbc-data-warehouse-unreturned-connection-timeout-seconds nil]
+ (binding [driver.settings/*query-timeout-ms* (u/minutes->ms 20)]
+ (testing "outside `with-transform-connection`, the default is db-query-timeout in seconds"
+ (is (= (* 20 60)
+ (driver.settings/jdbc-data-warehouse-unreturned-connection-timeout-seconds))))
+ (testing "inside `with-transform-connection` with the transform rebinding *query-timeout-ms*, the default
+ rises to transform-timeout in seconds — so the `:transform` pool created in this scope picks up
+ the longer leak-detector"
+ (driver.conn/with-transform-connection
+ (binding [driver.settings/*query-timeout-ms* (u/minutes->ms 240)]
+ (is (= (* 240 60)
+ (driver.settings/jdbc-data-warehouse-unreturned-connection-timeout-seconds))))))
+ (testing "an explicit env-var/setting override still wins over the computed default"
+ (mt/with-temporary-setting-values [jdbc-data-warehouse-unreturned-connection-timeout-seconds 15]
+ (driver.conn/with-transform-connection
+ (binding [driver.settings/*query-timeout-ms* (u/minutes->ms 240)]
+ (is (= 15 (driver.settings/jdbc-data-warehouse-unreturned-connection-timeout-seconds)))))))))))
+
+(deftest transform-connection-type-is-distinct-pool-key-test
+ (testing "The pool cache key derived from `*connection-type*` distinguishes `:transform` from `:default`, so the
+ two contexts get separate c3p0 pools with separate properties. This is the core mechanism that keeps the
+ default pool's leak detector tight on non-transform instances."
+ (mt/with-temp [:model/Database db {:engine :h2}]
+ (is (= :default
+ (driver.conn/effective-connection-type db)))
+ (driver.conn/with-transform-connection
+ (is (= :transform
+ (driver.conn/effective-connection-type db)))))))
+
+;; Not ^:parallel: the kondo linter flags `set-statement-query-timeout!` (`!`-suffixed, so classified "destructive")
+;; when used inside a parallel test. The call site here only mutates a local proxy Statement and is safe, but marking
+;; synchronous avoids growing the whitelist.
+(deftest set-statement-query-timeout!-test
+ (testing "the helper that populates Statement.setQueryTimeout reads *query-timeout-ms* and converts to seconds.
+ Proved via a mock Statement so the test is deterministic and independent of any driver's enforcement
+ semantics."
+ (let [captured-seconds (atom nil)
+ mock-stmt (proxy [java.sql.Statement] []
+ (setQueryTimeout [secs] (reset! captured-seconds secs)))
+ set-timeout! @#'sql-jdbc.execute/set-statement-query-timeout!]
+ (testing "default dynamic scope"
+ (binding [driver.settings/*query-timeout-ms* (u/minutes->ms 3)]
+ (set-timeout! :h2 mock-stmt)
+ (is (= (* 3 60) @captured-seconds))))
+ (testing "transform-scope rebinding lands in the Statement"
+ (binding [driver.settings/*query-timeout-ms* (u/minutes->ms 90)]
+ (set-timeout! :h2 mock-stmt)
+ (is (= (* 90 60) @captured-seconds))))
+ (testing "a throwing driver does not propagate the exception"
+ (let [throwing-stmt (proxy [java.sql.Statement] []
+ (setQueryTimeout [_] (throw (java.sql.SQLFeatureNotSupportedException.))))]
+ (is (nil? (set-timeout! :h2 throwing-stmt)))))
+ (testing "drivers that opt out via :jdbc/set-query-timeout=false skip the call entirely"
+ (reset! captured-seconds :not-called)
+ (with-redefs [driver/database-supports? (fn [_ feature _] (not= feature :jdbc/set-query-timeout))]
+ (binding [driver.settings/*query-timeout-ms* (u/minutes->ms 3)]
+ (set-timeout! :sparksql mock-stmt)
+ (is (= :not-called @captured-seconds))))))))
+
+(deftest statement-or-prepared-statement-round-trips-query-timeout-test
+ (testing "statement-or-prepared-statement creates a Statement whose .getQueryTimeout reflects the currently
+ bound *query-timeout-ms*, for every SQL-JDBC driver in CI that supports `:jdbc/set-query-timeout`.
+ Drivers that opt out (e.g. SparkSQL — Hive's Thrift transport breaks if setQueryTimeout is called) are
+ excluded; we just verify the Statement was created."
+ (mt/test-drivers (into #{} (filter #(isa? driver/hierarchy % :sql-jdbc)) (mt/normal-drivers))
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver/*driver* (mt/db) {:write? false}
+ (fn [^java.sql.Connection conn]
+ (doseq [minutes [3 90]]
+ (binding [driver.settings/*query-timeout-ms* (u/minutes->ms minutes)]
+ (with-open [^java.sql.Statement stmt (sql-jdbc.execute/statement-or-prepared-statement
+ driver/*driver* conn "SELECT 1" [] nil)]
+ (if (driver/database-supports? driver/*driver* :jdbc/set-query-timeout nil)
+ (is (= (* minutes 60) (.getQueryTimeout stmt))
+ (str "driver " driver/*driver* " did not round-trip " minutes "min via setQueryTimeout"))
+ (is (some? stmt)
+ (str "driver " driver/*driver* " opted out of :jdbc/set-query-timeout; just confirming a Statement was returned")))))))))))
+
+(deftest run-cancelable-transform!-propagates-timeout-to-driver-test
+ (testing "run-cancelable-transform! rebinds *query-timeout-ms* for the whole transform body, so any driver that
+ reads the dynamic var at query time (SQL JDBC via setQueryTimeout, Mongo/Druid/BigQuery directly) sees
+ the transform timeout instead of db-query-timeout."
+ (let [driver-observed-timeout-ms (atom nil)]
+ (with-redefs [driver/schema-exists? (constantly true)
+ driver/create-schema-if-needed! (constantly nil)
+ transforms-base.u/get-source-range-params (constantly nil)
+ transforms-base.u/save-run-checkpoint-range! (constantly nil)
+ transforms-base.u/save-watermark! (constantly nil)
+ transforms.canceling/chan-start-timeout-vthread! (constantly nil)
+ transforms.canceling/chan-start-run! (constantly nil)
+ transforms.canceling/chan-end-run! (constantly nil)
+ transform-run/succeed-started-run! (constantly nil)]
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-setting-values [transform-timeout 90]
+ (transforms.u/run-cancelable-transform!
+ 1 {:id 1} :h2 {:db-id 1 :conn-spec nil :output-schema "x"}
+ (fn [_cancel-chan _range-params]
+ (reset! driver-observed-timeout-ms driver.settings/*query-timeout-ms*))))))
+ (is (= (u/minutes->ms 90) @driver-observed-timeout-ms)))))
+
(deftest ^:parallel massage-sql-query-test
(testing "massage-sql-query sets disable-remaps? and disable-max-results?"
(let [query {:database 1, :type :query, :query {:source-table 1}}