Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Revert "Enable streaming results for app db queries with postgres" (#77048)

Alexander Polyankinaf81edb34651
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index d4a984b32c8..93b7fec0e88 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2497,7 +2497,8 @@
:model/QueryExecution
:model/Segment
:model/Table}
- :uses #{lib
+ :uses #{app-db
+ lib
lib-be
models
settings
diff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.clj
index 4645c54bbf5..ce2c4edead0 100644
--- a/src/metabase/app_db/connection.clj
+++ b/src/metabase/app_db/connection.clj
@@ -9,7 +9,6 @@
[potemkin :as p]
[toucan2.connection :as t2.conn]
[toucan2.jdbc.connection :as t2.jdbc.conn]
- [toucan2.jdbc.options :as t2.jdbc.options]
[toucan2.pipeline :as t2.pipeline])
(:import
(java.util.concurrent.locks ReentrantReadWriteLock)))
@@ -129,63 +128,9 @@
[]
(.id *application-db*))
-(defn- reset-autocommit!
- "Restore `connection`'s autoCommit to the JDBC default of true, logging -- but not propagating -- any failure, so a
- reset error never masks the original exception or breaks the connection's return to the pool."
- [^java.sql.Connection connection]
- (try
- (.setAutoCommit connection true)
- (catch Throwable t
- (log/warn t "Failed to reset the connection's autocommit flag to true"))))
-
-(def ^:private select-fetch-size
- "Default JDBC fetch size (rows per network round-trip) for app-db queries. Postgres only streams a result set from a
- server-side cursor -- instead of materializing the whole thing in client memory -- when the fetch size is positive
- *and* autoCommit is off (see [[do-with-app-db-connection]]); the JDBC default fetch size of 0 means \"fetch
- everything\"."
- 500)
-
-(defn- do-with-app-db-connection
- "Acquire a connection from the application DB and run `f` on it.
-
- On Postgres we run the connection with autoCommit *off* so the driver streams large SELECTs from a server-side cursor
- instead of buffering them into client memory, and we commit (or roll back on error) manually at the end of the
- connection scope. We also set a positive default fetch size, which Postgres additionally requires before it will use a
- cursor. Writes still commit -- Toucan 2's transaction handling commits at the end of each [[t2/with-transaction]], and
- anything left uncommitted in the scope (e.g. a bare SELECT) is committed here.
-
- We restore autoCommit=true before the connection returns to the pool: other borrowers of the same pooled connection
- (Liquibase, raw `.getConnection` callers) expect the JDBC default, and in particular the pool's `on-check-in` hook
- runs `DISCARD ALL`, which Postgres refuses to run inside a transaction block. H2/MySQL keep the plain JDBC-default
- behavior.
-
- This runs only at the outermost connection acquisition (nested Toucan calls reuse the already-bound
- `java.sql.Connection`), so `conn` is always a freshly checked-out pooled connection we own outright. We flip autoCommit
- off unconditionally -- idempotent if it is somehow already off -- and always commit and reset, which also heals a
- connection a previous borrower may have left in manual-commit mode."
- [^ApplicationDB app-db f]
- (if (not= :postgres (.db-type app-db))
- (with-open [conn (.getConnection app-db)]
- (f conn))
- (binding [t2.jdbc.options/*options* (assoc t2.jdbc.options/*options* :fetch-size select-fetch-size)]
- (with-open [conn (.getConnection app-db)]
- (.setAutoCommit conn false)
- (try
- (let [result (f conn)]
- (.commit conn)
- result)
- (catch Throwable e
- (try
- (.rollback conn)
- (catch Throwable rollback-e
- (log/warn rollback-e "Failed to roll back app-db connection")))
- (throw e))
- (finally
- (reset-autocommit! conn)))))))
-
(methodical/defmethod t2.conn/do-with-connection :default
[_connectable f]
- (do-with-app-db-connection *application-db* f))
+ (t2.conn/do-with-connection *application-db* f))
(def ^:private ^:dynamic *transaction-depth* 0)
@@ -263,7 +208,10 @@
(thunk)
(finally
;; prevent a failing .setAutoCommit call from masking the original exception
- (reset-autocommit! connection)))
+ (try
+ (.setAutoCommit connection true)
+ (catch Throwable t
+ (log/warn t "Failed to reset the connection's autocommit flag to true")))))
(thunk))))
(comment
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index 50f1cb594ff..9cb88405f89 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -67,6 +67,8 @@
qualify
query
select-or-insert!
+ streaming-reducible
+ streaming-reducible-query
type-keyword->descendants
update-or-insert!
with-conflict-retry]
diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.clj
index 65661a78643..5383196c3a8 100644
--- a/src/metabase/app_db/query.clj
+++ b/src/metabase/app_db/query.clj
@@ -147,6 +147,40 @@
:uncompiled sql-args-or-honey-sql-map}
e)))))))
+(defn streaming-reducible
+ "Returns a reducible that, when reduced, opens a transaction (autoCommit=false) with a
+ fetch-size set to enable server-side cursor streaming on PostgreSQL app databases.
+
+ `make-reducible-fn` is a 1-arg function called with the transaction `java.sql.Connection`.
+ Pass it explicitly to `t2/reducible-select` via `:conn` or to `t2/reducible-query` so the
+ query runs on the transaction connection — otherwise PostgreSQL ignores fetch-size.
+
+ WARNING: Because this opens a transaction, do not wrap any long-running logic in this which will hold the transaction open too long.
+
+ Example:
+ (streaming-reducible
+ (fn [conn]
+ (eduction (map t2.realize/realize)
+ (t2/reducible-select :conn conn :model/Table :active true))))"
+ ([make-reducible-fn]
+ (streaming-reducible make-reducible-fn 500))
+ ([make-reducible-fn fetch-size]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ f init]
+ (t2/with-transaction [conn]
+ (binding [t2.jdbc.options/*options* (merge t2.jdbc.options/*options*
+ {:fetch-size fetch-size})]
+ (reduce f init (make-reducible-fn conn))))))))
+
+(defn streaming-reducible-query
+ "Like `t2/reducible-query` but uses a transaction (autoCommit=false) and a fetch-size
+ to enable server-side cursor streaming on PostgreSQL. Returns a reducible of raw maps."
+ ([sql-args-or-honey-sql-map]
+ (streaming-reducible-query sql-args-or-honey-sql-map 500))
+ ([sql-args-or-honey-sql-map fetch-size]
+ (let [compiled (compile sql-args-or-honey-sql-map)]
+ (streaming-reducible (fn [conn] (t2/reducible-query conn compiled)) fetch-size))))
+
(defmacro with-conflict-retry
"Retry a database mutation a single time if it fails due to concurrent insertions.
May retry for other reasons."
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index 0a4c33476c1..90b64c37133 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -719,7 +719,7 @@
(log/tracef "Searching with query:\n%s\n%s"
(u/pprint-to-str search-query)
(mdb/format-sql (first (mdb/compile search-query))))
- (t2/reducible-query search-query)))
+ (mdb/streaming-reducible-query search-query)))
(defmethod search.engine/results
:search.engine/in-place
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 22ddde27e28..e5a112ab9ca 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -210,7 +210,7 @@
;; joined side has its own integrity violations. Downstream upserts hit a unique constraint on
;; (model, model_id), so dedup here at the streaming boundary — bounded by per-model row count.
(->> (spec-index-query-where search-model where-clause)
- t2/reducible-query
+ mdb/streaming-reducible-query
(eduction (comp (map #(assoc % :model search-model))
(m/distinct-by (juxt :id :model))))))
diff --git a/src/metabase/usage_metadata/batch.clj b/src/metabase/usage_metadata/batch.clj
index c2ac4ff08e9..8863db4f088 100644
--- a/src/metabase/usage_metadata/batch.clj
+++ b/src/metabase/usage_metadata/batch.clj
@@ -1,6 +1,7 @@
(ns metabase.usage-metadata.batch
(:require
[java-time.api :as t]
+ [metabase.app-db.core :as mdb]
[metabase.lib-be.core :as lib-be]
[metabase.usage-metadata.extract :as usage-metadata.extract]
[metabase.usage-metadata.models.source-dimension-daily]
@@ -282,8 +283,10 @@
(map t2.realize/realize)
(completing (partial process-query-row bucket-date hash->count))
stats
- (t2/reducible-select [:model/Query :query_hash :query]
- :query_hash [:in hash-chunk])))
+ (mdb/streaming-reducible
+ (fn [conn]
+ (t2/reducible-select :conn conn [:model/Query :query_hash :query]
+ :query_hash [:in hash-chunk])))))
initial-stats
(partition-all hash-chunk-size raw-hashes))
seen-hashes (:seen-hashes after-stream)
diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.clj
index 2a3adcc4136..a4d59faccce 100644
--- a/test/metabase/app_db/connection_test.clj
+++ b/test/metabase/app_db/connection_test.clj
@@ -8,8 +8,7 @@
[toucan2.core :as t2])
(:import
(java.sql Connection)
- (java.util.concurrent Semaphore)
- (javax.sql DataSource)))
+ (java.util.concurrent Semaphore)))
(set! *warn-on-reflection* true)
@@ -63,10 +62,7 @@
(t2/with-transaction [_t-conn conn]
;; dummy op
(is (false? (.getAutoCommit conn))))
- ;; On a Postgres app DB the connection runs with autoCommit off for its whole scope (so SELECTs stream from a
- ;; cursor); on H2/MySQL the transaction restores the JDBC default of true.
- (is (= (not= (mdb.connection/db-type) :postgres)
- (.getAutoCommit conn)))))
+ (is (true? (.getAutoCommit conn)))))
(testing "throw error when trying to create nested transaction when nested-transaction-rule=:prohibit"
(t2/with-connection [conn]
(t2/with-transaction [t-conn conn]
@@ -248,83 +244,3 @@
;; The original exception should be thrown, not the setAutoCommit exception
(is (= msg (ex-message e))))
(is (true? @autocommit-reset-called))))))
-
-(deftest ^:synchronized reducible-query-streams-large-result-set-test
- (testing "when using a Postgres app DB, [[t2/reducible-query]] streams via a server-side cursor (autoCommit=false + fetch size)"
- (when (= (mdb.connection/db-type) :postgres)
- (let [n Integer/MAX_VALUE
- result (t2/reducible-query [(format "SELECT generate_series(1, %d) AS i" n)])]
- (testing "only the first rows are pulled, not all ~2 billion"
- (is (= [1 2 3] (into [] (comp (take 3) (map :i)) result))))))))
-
-(deftest ^:synchronized postgres-app-db-runs-with-autocommit-off-test
- (testing "when using a Postgres app DB, toucan2 connections run with autoCommit off and we commit manually"
- (when (= (mdb.connection/db-type) :postgres)
- (testing "a connection handed out by toucan2 has autoCommit disabled for its whole scope"
- (t2/with-connection [^java.sql.Connection conn]
- (is (false? (.getAutoCommit conn)))))
- (testing "writes are still committed (manually, at the end of the connection scope) and survive"
- (let [email (mt/random-email)]
- (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))
- (is (true? (t2/exists? :model/User :email email))))))))
-
-(defn- recording-connection
- "A bare [[Connection]] that starts at `initial-autocommit` and appends a marker for each setAutoCommit/commit/rollback/
- close call to the `calls` volatile (holding a vector). Lets us assert the exact side effects
- [[mdb.connection/do-with-app-db-connection]] performs against a connection without needing a real database."
- ^Connection [initial-autocommit calls]
- (let [autocommit (volatile! initial-autocommit)]
- (reify Connection
- (getAutoCommit [_] @autocommit)
- (setAutoCommit [_ v] (vswap! calls conj [:set-autocommit v]) (vreset! autocommit v))
- (commit [_] (vswap! calls conj :commit))
- (rollback [_] (vswap! calls conj :rollback))
- (close [_] (vswap! calls conj :close)))))
-
-(defn- mock-app-db
- "An [[mdb.connection/ApplicationDB]] of `db-type` (no pool) that always hands out `conn`."
- [db-type ^Connection conn]
- (mdb.connection/application-db db-type (reify DataSource (getConnection [_] conn))))
-
-(deftest do-with-app-db-connection-postgres-test
- (testing "when using a Postgres app DB, we flip autoCommit off for the scope, then commit/rollback and reset it before returning to the pool"
- (testing "happy path: autoCommit off -> commit -> reset to true -> close"
- (let [calls (volatile! [])
- conn (recording-connection true calls)
- result (#'mdb.connection/do-with-app-db-connection
- (mock-app-db :postgres conn)
- (fn [^Connection c]
- (is (false? (.getAutoCommit c)) "autoCommit is off while f runs")
- :result))]
- (is (= :result result))
- (is (= [[:set-autocommit false] :commit [:set-autocommit true] :close] @calls))))
- (testing "error path: rolls back, still resets autoCommit, and propagates the original exception"
- (let [calls (volatile! [])
- conn (recording-connection true calls)
- boom (ex-info "boom" {})]
- (is (identical? boom
- (try
- (#'mdb.connection/do-with-app-db-connection
- (mock-app-db :postgres conn)
- (fn [_] (throw boom)))
- (catch clojure.lang.ExceptionInfo e e))))
- (is (= [[:set-autocommit false] :rollback [:set-autocommit true] :close] @calls))))
- (testing "a connection that comes back already in manual-commit mode is taken over and reset (self-healing)"
- (let [calls (volatile! [])
- conn (recording-connection false calls)]
- (is (= :result
- (#'mdb.connection/do-with-app-db-connection
- (mock-app-db :postgres conn)
- (fn [_] :result))))
- (is (= [[:set-autocommit false] :commit [:set-autocommit true] :close] @calls))))))
-
-(deftest do-with-app-db-connection-non-postgres-test
- (testing "when using a non-Postgres app DB, autoCommit is never touched -- the connection is just used and closed"
- (doseq [db-type [:h2 :mysql]]
- (let [calls (volatile! [])
- conn (recording-connection true calls)]
- (is (= :result
- (#'mdb.connection/do-with-app-db-connection
- (mock-app-db db-type conn)
- (fn [_] :result))))
- (is (= [:close] @calls) (str "db-type " db-type))))))