Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Enable streaming results for app db queries with postgres (#76645)

Alexander Polyankin49cc5998921d
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 9dc51872f44..8c9cc4f593e 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2478,8 +2478,7 @@
:model/QueryExecution
:model/Segment
:model/Table}
- :uses #{app-db
- lib
+ :uses #{lib
lib-be
models
settings
diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.clj
index 8bd856c02fb..d67e05ed763 100644
--- a/src/metabase/app_db/connection.clj
+++ b/src/metabase/app_db/connection.clj
@@ -9,6 +9,7 @@
[potemkin :as p]
[toucan2.connection :as t2.conn]
[toucan2.jdbc.connection :as t2.jdbc.conn]
+ [toucan2.jdbc.options :as t2.jdbc.options]
[toucan2.pipeline :as t2.pipeline])
(:import
(java.util.concurrent.locks ReentrantReadWriteLock)))
@@ -128,9 +129,63 @@
[]
(.id *application-db*))
+(defn- reset-autocommit!
+ "Restore `connection`'s autoCommit to the JDBC default of true, logging -- but not propagating -- any failure, so a
+ reset error never masks the original exception or breaks the connection's return to the pool."
+ [^java.sql.Connection connection]
+ (try
+ (.setAutoCommit connection true)
+ (catch Throwable t
+ (log/warn t "Failed to reset the connection's autocommit flag to true"))))
+
+(def ^:private select-fetch-size
+ "Default JDBC fetch size (rows per network round-trip) for app-db queries. Postgres only streams a result set from a
+ server-side cursor -- instead of materializing the whole thing in client memory -- when the fetch size is positive
+ *and* autoCommit is off (see [[do-with-app-db-connection]]); the JDBC default fetch size of 0 means \"fetch
+ everything\"."
+ 500)
+
+(defn- do-with-app-db-connection
+ "Acquire a connection from the application DB and run `f` on it.
+
+ On Postgres we run the connection with autoCommit *off* so the driver streams large SELECTs from a server-side cursor
+ instead of buffering them into client memory, and we commit (or roll back on error) manually at the end of the
+ connection scope. We also set a positive default fetch size, which Postgres additionally requires before it will use a
+ cursor. Writes still commit -- Toucan 2's transaction handling commits at the end of each [[t2/with-transaction]], and
+ anything left uncommitted in the scope (e.g. a bare SELECT) is committed here.
+
+ We restore autoCommit=true before the connection returns to the pool: other borrowers of the same pooled connection
+ (Liquibase, raw `.getConnection` callers) expect the JDBC default, and in particular the pool's `on-check-in` hook
+ runs `DISCARD ALL`, which Postgres refuses to run inside a transaction block. H2/MySQL keep the plain JDBC-default
+ behavior.
+
+ This runs only at the outermost connection acquisition (nested Toucan calls reuse the already-bound
+ `java.sql.Connection`), so `conn` is always a freshly checked-out pooled connection we own outright. We flip autoCommit
+ off unconditionally -- idempotent if it is somehow already off -- and always commit and reset, which also heals a
+ connection a previous borrower may have left in manual-commit mode."
+ [^ApplicationDB app-db f]
+ (if (not= :postgres (.db-type app-db))
+ (with-open [conn (.getConnection app-db)]
+ (f conn))
+ (binding [t2.jdbc.options/*options* (assoc t2.jdbc.options/*options* :fetch-size select-fetch-size)]
+ (with-open [conn (.getConnection app-db)]
+ (.setAutoCommit conn false)
+ (try
+ (let [result (f conn)]
+ (.commit conn)
+ result)
+ (catch Throwable e
+ (try
+ (.rollback conn)
+ (catch Throwable rollback-e
+ (log/warn rollback-e "Failed to roll back app-db connection")))
+ (throw e))
+ (finally
+ (reset-autocommit! conn)))))))
+
(methodical/defmethod t2.conn/do-with-connection :default
[_connectable f]
- (t2.conn/do-with-connection *application-db* f))
+ (do-with-app-db-connection *application-db* f))
(def ^:private ^:dynamic *transaction-depth* 0)
@@ -164,10 +219,7 @@
(thunk)
(finally
;; prevent a failing .setAutoCommit call from masking the original exception
- (try
- (.setAutoCommit connection true)
- (catch Throwable t
- (log/warn t "Failed to reset the connection's autocommit flag to true")))))
+ (reset-autocommit! connection)))
(thunk))))
(comment
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index cf38535a856..1074a3d58a0 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -66,8 +66,6 @@
qualify
query
select-or-insert!
- streaming-reducible
- streaming-reducible-query
type-keyword->descendants
update-or-insert!
with-conflict-retry]
diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.clj
index 797fc468cb5..51303e02bd4 100644
--- a/src/metabase/app_db/query.clj
+++ b/src/metabase/app_db/query.clj
@@ -147,40 +147,6 @@
:uncompiled sql-args-or-honey-sql-map}
e)))))))
-(defn streaming-reducible
- "Returns a reducible that, when reduced, opens a transaction (autoCommit=false) with a
- fetch-size set to enable server-side cursor streaming on PostgreSQL app databases.
-
- `make-reducible-fn` is a 1-arg function called with the transaction `java.sql.Connection`.
- Pass it explicitly to `t2/reducible-select` via `:conn` or to `t2/reducible-query` so the
- query runs on the transaction connection — otherwise PostgreSQL ignores fetch-size.
-
- WARNING: Because this opens a transaction, do not wrap any long-running logic in this which will hold the transaction open too long.
-
- Example:
- (streaming-reducible
- (fn [conn]
- (eduction (map t2.realize/realize)
- (t2/reducible-select :conn conn :model/Table :active true))))"
- ([make-reducible-fn]
- (streaming-reducible make-reducible-fn 500))
- ([make-reducible-fn fetch-size]
- (reify clojure.lang.IReduceInit
- (reduce [_ f init]
- (t2/with-transaction [conn]
- (binding [t2.jdbc.options/*options* (merge t2.jdbc.options/*options*
- {:fetch-size fetch-size})]
- (reduce f init (make-reducible-fn conn))))))))
-
-(defn streaming-reducible-query
- "Like `t2/reducible-query` but uses a transaction (autoCommit=false) and a fetch-size
- to enable server-side cursor streaming on PostgreSQL. Returns a reducible of raw maps."
- ([sql-args-or-honey-sql-map]
- (streaming-reducible-query sql-args-or-honey-sql-map 500))
- ([sql-args-or-honey-sql-map fetch-size]
- (let [compiled (compile sql-args-or-honey-sql-map)]
- (streaming-reducible (fn [conn] (t2/reducible-query conn compiled)) fetch-size))))
-
(defmacro with-conflict-retry
"Retry a database mutation a single time if it fails due to concurrent insertions.
May retry for other reasons."
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index 90b64c37133..0a4c33476c1 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -719,7 +719,7 @@
(log/tracef "Searching with query:\n%s\n%s"
(u/pprint-to-str search-query)
(mdb/format-sql (first (mdb/compile search-query))))
- (mdb/streaming-reducible-query search-query)))
+ (t2/reducible-query search-query)))
(defmethod search.engine/results
:search.engine/in-place
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index e5a112ab9ca..22ddde27e28 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -210,7 +210,7 @@
;; joined side has its own integrity violations. Downstream upserts hit a unique constraint on
;; (model, model_id), so dedup here at the streaming boundary — bounded by per-model row count.
(->> (spec-index-query-where search-model where-clause)
- mdb/streaming-reducible-query
+ t2/reducible-query
(eduction (comp (map #(assoc % :model search-model))
(m/distinct-by (juxt :id :model))))))
diff --git a/src/metabase/usage_metadata/batch.clj b/src/metabase/usage_metadata/batch.clj
index 8863db4f088..c2ac4ff08e9 100644
--- a/src/metabase/usage_metadata/batch.clj
+++ b/src/metabase/usage_metadata/batch.clj
@@ -1,7 +1,6 @@
(ns metabase.usage-metadata.batch
(:require
[java-time.api :as t]
- [metabase.app-db.core :as mdb]
[metabase.lib-be.core :as lib-be]
[metabase.usage-metadata.extract :as usage-metadata.extract]
[metabase.usage-metadata.models.source-dimension-daily]
@@ -283,10 +282,8 @@
(map t2.realize/realize)
(completing (partial process-query-row bucket-date hash->count))
stats
- (mdb/streaming-reducible
- (fn [conn]
- (t2/reducible-select :conn conn [:model/Query :query_hash :query]
- :query_hash [:in hash-chunk])))))
+ (t2/reducible-select [:model/Query :query_hash :query]
+ :query_hash [:in hash-chunk])))
initial-stats
(partition-all hash-chunk-size raw-hashes))
seen-hashes (:seen-hashes after-stream)
diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.clj
index c1b1ad70195..68a24965195 100644
--- a/test/metabase/app_db/connection_test.clj
+++ b/test/metabase/app_db/connection_test.clj
@@ -8,7 +8,8 @@
[toucan2.core :as t2])
(:import
(java.sql Connection)
- (java.util.concurrent Semaphore)))
+ (java.util.concurrent Semaphore)
+ (javax.sql DataSource)))
(set! *warn-on-reflection* true)
@@ -62,7 +63,10 @@
(t2/with-transaction [_t-conn conn]
;; dummy op
(is (false? (.getAutoCommit conn))))
- (is (true? (.getAutoCommit conn)))))
+ ;; On a Postgres app DB the connection runs with autoCommit off for its whole scope (so SELECTs stream from a
+ ;; cursor); on H2/MySQL the transaction restores the JDBC default of true.
+ (is (= (not= (mdb.connection/db-type) :postgres)
+ (.getAutoCommit conn)))))
(testing "throw error when trying to create nested transaction when nested-transaction-rule=:prohibit"
(t2/with-connection [conn]
(t2/with-transaction [t-conn conn]
@@ -172,3 +176,83 @@
;; The original exception should be thrown, not the setAutoCommit exception
(is (= msg (ex-message e))))
(is (true? @autocommit-reset-called))))))
+
+(deftest ^:synchronized reducible-query-streams-large-result-set-test
+ (testing "when using a Postgres app DB, [[t2/reducible-query]] streams via a server-side cursor (autoCommit=false + fetch size)"
+ (when (= (mdb.connection/db-type) :postgres)
+ (let [n Integer/MAX_VALUE
+ result (t2/reducible-query [(format "SELECT generate_series(1, %d) AS i" n)])]
+ (testing "only the first rows are pulled, not all ~2 billion"
+ (is (= [1 2 3] (into [] (comp (take 3) (map :i)) result))))))))
+
+(deftest ^:synchronized postgres-app-db-runs-with-autocommit-off-test
+ (testing "when using a Postgres app DB, toucan2 connections run with autoCommit off and we commit manually"
+ (when (= (mdb.connection/db-type) :postgres)
+ (testing "a connection handed out by toucan2 has autoCommit disabled for its whole scope"
+ (t2/with-connection [^java.sql.Connection conn]
+ (is (false? (.getAutoCommit conn)))))
+ (testing "writes are still committed (manually, at the end of the connection scope) and survive"
+ (let [email (mt/random-email)]
+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))
+ (is (true? (t2/exists? :model/User :email email))))))))
+
+(defn- recording-connection
+ "A bare [[Connection]] that starts at `initial-autocommit` and appends a marker for each setAutoCommit/commit/rollback/
+ close call to the `calls` volatile (holding a vector). Lets us assert the exact side effects
+ [[mdb.connection/do-with-app-db-connection]] performs against a connection without needing a real database."
+ ^Connection [initial-autocommit calls]
+ (let [autocommit (volatile! initial-autocommit)]
+ (reify Connection
+ (getAutoCommit [_] @autocommit)
+ (setAutoCommit [_ v] (vswap! calls conj [:set-autocommit v]) (vreset! autocommit v))
+ (commit [_] (vswap! calls conj :commit))
+ (rollback [_] (vswap! calls conj :rollback))
+ (close [_] (vswap! calls conj :close)))))
+
+(defn- mock-app-db
+ "An [[mdb.connection/ApplicationDB]] of `db-type` (no pool) that always hands out `conn`."
+ [db-type ^Connection conn]
+ (mdb.connection/application-db db-type (reify DataSource (getConnection [_] conn))))
+
+(deftest do-with-app-db-connection-postgres-test
+ (testing "when using a Postgres app DB, we flip autoCommit off for the scope, then commit/rollback and reset it before returning to the pool"
+ (testing "happy path: autoCommit off -> commit -> reset to true -> close"
+ (let [calls (volatile! [])
+ conn (recording-connection true calls)
+ result (#'mdb.connection/do-with-app-db-connection
+ (mock-app-db :postgres conn)
+ (fn [^Connection c]
+ (is (false? (.getAutoCommit c)) "autoCommit is off while f runs")
+ :result))]
+ (is (= :result result))
+ (is (= [[:set-autocommit false] :commit [:set-autocommit true] :close] @calls))))
+ (testing "error path: rolls back, still resets autoCommit, and propagates the original exception"
+ (let [calls (volatile! [])
+ conn (recording-connection true calls)
+ boom (ex-info "boom" {})]
+ (is (identical? boom
+ (try
+ (#'mdb.connection/do-with-app-db-connection
+ (mock-app-db :postgres conn)
+ (fn [_] (throw boom)))
+ (catch clojure.lang.ExceptionInfo e e))))
+ (is (= [[:set-autocommit false] :rollback [:set-autocommit true] :close] @calls))))
+ (testing "a connection that comes back already in manual-commit mode is taken over and reset (self-healing)"
+ (let [calls (volatile! [])
+ conn (recording-connection false calls)]
+ (is (= :result
+ (#'mdb.connection/do-with-app-db-connection
+ (mock-app-db :postgres conn)
+ (fn [_] :result))))
+ (is (= [[:set-autocommit false] :commit [:set-autocommit true] :close] @calls))))))
+
+(deftest do-with-app-db-connection-non-postgres-test
+ (testing "when using a non-Postgres app DB, autoCommit is never touched -- the connection is just used and closed"
+ (doseq [db-type [:h2 :mysql]]
+ (let [calls (volatile! [])
+ conn (recording-connection true calls)]
+ (is (= :result
+ (#'mdb.connection/do-with-app-db-connection
+ (mock-app-db db-type conn)
+ (fn [_] :result))))
+ (is (= [:close] @calls) (str "db-type " db-type))))))