The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Enable streaming results for app db queries with postgres (#76645)
Alexander Polyankin49cc5998921d
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 9dc51872f44..8c9cc4f593e 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2478,8 +2478,7 @@:model/QueryExecution:model/Segment:model/Table}- :uses #{app-db- lib+ :uses #{liblib-bemodelssettingsdiff --git a/src/metabase/app_db/connection.clj b/src/metabase/app_db/connection.cljindex 8bd856c02fb..d67e05ed763 100644--- a/src/metabase/app_db/connection.clj+++ b/src/metabase/app_db/connection.clj@@ -9,6 +9,7 @@[potemkin :as p][toucan2.connection :as t2.conn][toucan2.jdbc.connection :as t2.jdbc.conn]+ [toucan2.jdbc.options :as t2.jdbc.options][toucan2.pipeline :as t2.pipeline])(:import(java.util.concurrent.locks ReentrantReadWriteLock)))@@ -128,9 +129,63 @@[](.id *application-db*))+(defn- reset-autocommit!+ "Restore `connection`'s autoCommit to the JDBC default of true, logging -- but not propagating -- any failure, so a+ reset error never masks the original exception or breaks the connection's return to the pool."+ [^java.sql.Connection connection]+ (try+ (.setAutoCommit connection true)+ (catch Throwable t+ (log/warn t "Failed to reset the connection's autocommit flag to true"))))++(def ^:private select-fetch-size+ "Default JDBC fetch size (rows per network round-trip) for app-db queries. Postgres only streams a result set from a+ server-side cursor -- instead of materializing the whole thing in client memory -- when the fetch size is positive+ *and* autoCommit is off (see [[do-with-app-db-connection]]); the JDBC default fetch size of 0 means \"fetch+ everything\"."+ 500)++(defn- do-with-app-db-connection+ "Acquire a connection from the application DB and run `f` on it.++ On Postgres we run the connection with autoCommit *off* so the driver streams large SELECTs from a server-side cursor+ instead of buffering them into client memory, and we commit (or roll back on error) manually at the end of the+ connection scope. We also set a positive default fetch size, which Postgres additionally requires before it will use a+ cursor. Writes still commit -- Toucan 2's transaction handling commits at the end of each [[t2/with-transaction]], and+ anything left uncommitted in the scope (e.g. a bare SELECT) is committed here.++ We restore autoCommit=true before the connection returns to the pool: other borrowers of the same pooled connection+ (Liquibase, raw `.getConnection` callers) expect the JDBC default, and in particular the pool's `on-check-in` hook+ runs `DISCARD ALL`, which Postgres refuses to run inside a transaction block. H2/MySQL keep the plain JDBC-default+ behavior.++ This runs only at the outermost connection acquisition (nested Toucan calls reuse the already-bound+ `java.sql.Connection`), so `conn` is always a freshly checked-out pooled connection we own outright. We flip autoCommit+ off unconditionally -- idempotent if it is somehow already off -- and always commit and reset, which also heals a+ connection a previous borrower may have left in manual-commit mode."+ [^ApplicationDB app-db f]+ (if (not= :postgres (.db-type app-db))+ (with-open [conn (.getConnection app-db)]+ (f conn))+ (binding [t2.jdbc.options/*options* (assoc t2.jdbc.options/*options* :fetch-size select-fetch-size)]+ (with-open [conn (.getConnection app-db)]+ (.setAutoCommit conn false)+ (try+ (let [result (f conn)]+ (.commit conn)+ result)+ (catch Throwable e+ (try+ (.rollback conn)+ (catch Throwable rollback-e+ (log/warn rollback-e "Failed to roll back app-db connection")))+ (throw e))+ (finally+ (reset-autocommit! conn)))))))+(methodical/defmethod t2.conn/do-with-connection :default[_connectable f]- (t2.conn/do-with-connection *application-db* f))+ (do-with-app-db-connection *application-db* f))(def ^:private ^:dynamic *transaction-depth* 0)@@ -164,10 +219,7 @@(thunk)(finally;; prevent a failing .setAutoCommit call from masking the original exception- (try- (.setAutoCommit connection true)- (catch Throwable t- (log/warn t "Failed to reset the connection's autocommit flag to true")))))+ (reset-autocommit! connection)))(thunk))))(commentdiff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex cf38535a856..1074a3d58a0 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -66,8 +66,6 @@qualifyqueryselect-or-insert!- streaming-reducible- streaming-reducible-querytype-keyword->descendantsupdate-or-insert!with-conflict-retry]diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.cljindex 797fc468cb5..51303e02bd4 100644--- a/src/metabase/app_db/query.clj+++ b/src/metabase/app_db/query.clj@@ -147,40 +147,6 @@:uncompiled sql-args-or-honey-sql-map}e)))))))-(defn streaming-reducible- "Returns a reducible that, when reduced, opens a transaction (autoCommit=false) with a- fetch-size set to enable server-side cursor streaming on PostgreSQL app databases.-- `make-reducible-fn` is a 1-arg function called with the transaction `java.sql.Connection`.- Pass it explicitly to `t2/reducible-select` via `:conn` or to `t2/reducible-query` so the- query runs on the transaction connection — otherwise PostgreSQL ignores fetch-size.-- WARNING: Because this opens a transaction, do not wrap any long-running logic in this which will hold the transaction open too long.-- Example:- (streaming-reducible- (fn [conn]- (eduction (map t2.realize/realize)- (t2/reducible-select :conn conn :model/Table :active true))))"- ([make-reducible-fn]- (streaming-reducible make-reducible-fn 500))- ([make-reducible-fn fetch-size]- (reify clojure.lang.IReduceInit- (reduce [_ f init]- (t2/with-transaction [conn]- (binding [t2.jdbc.options/*options* (merge t2.jdbc.options/*options*- {:fetch-size fetch-size})]- (reduce f init (make-reducible-fn conn))))))))--(defn streaming-reducible-query- "Like `t2/reducible-query` but uses a transaction (autoCommit=false) and a fetch-size- to enable server-side cursor streaming on PostgreSQL. Returns a reducible of raw maps."- ([sql-args-or-honey-sql-map]- (streaming-reducible-query sql-args-or-honey-sql-map 500))- ([sql-args-or-honey-sql-map fetch-size]- (let [compiled (compile sql-args-or-honey-sql-map)]- (streaming-reducible (fn [conn] (t2/reducible-query conn compiled)) fetch-size))))-(defmacro with-conflict-retry"Retry a database mutation a single time if it fails due to concurrent insertions.May retry for other reasons."diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.cljindex 90b64c37133..0a4c33476c1 100644--- a/src/metabase/search/in_place/legacy.clj+++ b/src/metabase/search/in_place/legacy.clj@@ -719,7 +719,7 @@(log/tracef "Searching with query:\n%s\n%s"(u/pprint-to-str search-query)(mdb/format-sql (first (mdb/compile search-query))))- (mdb/streaming-reducible-query search-query)))+ (t2/reducible-query search-query)))(defmethod search.engine/results:search.engine/in-placediff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex e5a112ab9ca..22ddde27e28 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -210,7 +210,7 @@;; joined side has its own integrity violations. Downstream upserts hit a unique constraint on;; (model, model_id), so dedup here at the streaming boundary — bounded by per-model row count.(->> (spec-index-query-where search-model where-clause)- mdb/streaming-reducible-query+ t2/reducible-query(eduction (comp (map #(assoc % :model search-model))(m/distinct-by (juxt :id :model))))))diff --git a/src/metabase/usage_metadata/batch.clj b/src/metabase/usage_metadata/batch.cljindex 8863db4f088..c2ac4ff08e9 100644--- a/src/metabase/usage_metadata/batch.clj+++ b/src/metabase/usage_metadata/batch.clj@@ -1,7 +1,6 @@(ns metabase.usage-metadata.batch(:require[java-time.api :as t]- [metabase.app-db.core :as mdb][metabase.lib-be.core :as lib-be][metabase.usage-metadata.extract :as usage-metadata.extract][metabase.usage-metadata.models.source-dimension-daily]@@ -283,10 +282,8 @@(map t2.realize/realize)(completing (partial process-query-row bucket-date hash->count))stats- (mdb/streaming-reducible- (fn [conn]- (t2/reducible-select :conn conn [:model/Query :query_hash :query]- :query_hash [:in hash-chunk])))))+ (t2/reducible-select [:model/Query :query_hash :query]+ :query_hash [:in hash-chunk])))initial-stats(partition-all hash-chunk-size raw-hashes))seen-hashes (:seen-hashes after-stream)diff --git a/test/metabase/app_db/connection_test.clj b/test/metabase/app_db/connection_test.cljindex c1b1ad70195..68a24965195 100644--- a/test/metabase/app_db/connection_test.clj+++ b/test/metabase/app_db/connection_test.clj@@ -8,7 +8,8 @@[toucan2.core :as t2])(:import(java.sql Connection)- (java.util.concurrent Semaphore)))+ (java.util.concurrent Semaphore)+ (javax.sql DataSource)))(set! *warn-on-reflection* true)@@ -62,7 +63,10 @@(t2/with-transaction [_t-conn conn];; dummy op(is (false? (.getAutoCommit conn))))- (is (true? (.getAutoCommit conn)))))+ ;; On a Postgres app DB the connection runs with autoCommit off for its whole scope (so SELECTs stream from a+ ;; cursor); on H2/MySQL the transaction restores the JDBC default of true.+ (is (= (not= (mdb.connection/db-type) :postgres)+ (.getAutoCommit conn)))))(testing "throw error when trying to create nested transaction when nested-transaction-rule=:prohibit"(t2/with-connection [conn](t2/with-transaction [t-conn conn]@@ -172,3 +176,83 @@;; The original exception should be thrown, not the setAutoCommit exception(is (= msg (ex-message e))))(is (true? @autocommit-reset-called))))))++(deftest ^:synchronized reducible-query-streams-large-result-set-test+ (testing "when using a Postgres app DB, [[t2/reducible-query]] streams via a server-side cursor (autoCommit=false + fetch size)"+ (when (= (mdb.connection/db-type) :postgres)+ (let [n Integer/MAX_VALUE+ result (t2/reducible-query [(format "SELECT generate_series(1, %d) AS i" n)])]+ (testing "only the first rows are pulled, not all ~2 billion"+ (is (= [1 2 3] (into [] (comp (take 3) (map :i)) result))))))))++(deftest ^:synchronized postgres-app-db-runs-with-autocommit-off-test+ (testing "when using a Postgres app DB, toucan2 connections run with autoCommit off and we commit manually"+ (when (= (mdb.connection/db-type) :postgres)+ (testing "a connection handed out by toucan2 has autoCommit disabled for its whole scope"+ (t2/with-connection [^java.sql.Connection conn]+ (is (false? (.getAutoCommit conn)))))+ (testing "writes are still committed (manually, at the end of the connection scope) and survive"+ (let [email (mt/random-email)]+ (t2/insert! :model/User (assoc (mt/with-temp-defaults :model/User) :email email))+ (is (true? (t2/exists? :model/User :email email))))))))++(defn- recording-connection+ "A bare [[Connection]] that starts at `initial-autocommit` and appends a marker for each setAutoCommit/commit/rollback/+ close call to the `calls` volatile (holding a vector). Lets us assert the exact side effects+ [[mdb.connection/do-with-app-db-connection]] performs against a connection without needing a real database."+ ^Connection [initial-autocommit calls]+ (let [autocommit (volatile! initial-autocommit)]+ (reify Connection+ (getAutoCommit [_] @autocommit)+ (setAutoCommit [_ v] (vswap! calls conj [:set-autocommit v]) (vreset! autocommit v))+ (commit [_] (vswap! calls conj :commit))+ (rollback [_] (vswap! calls conj :rollback))+ (close [_] (vswap! calls conj :close)))))++(defn- mock-app-db+ "An [[mdb.connection/ApplicationDB]] of `db-type` (no pool) that always hands out `conn`."+ [db-type ^Connection conn]+ (mdb.connection/application-db db-type (reify DataSource (getConnection [_] conn))))++(deftest do-with-app-db-connection-postgres-test+ (testing "when using a Postgres app DB, we flip autoCommit off for the scope, then commit/rollback and reset it before returning to the pool"+ (testing "happy path: autoCommit off -> commit -> reset to true -> close"+ (let [calls (volatile! [])+ conn (recording-connection true calls)+ result (#'mdb.connection/do-with-app-db-connection+ (mock-app-db :postgres conn)+ (fn [^Connection c]+ (is (false? (.getAutoCommit c)) "autoCommit is off while f runs")+ :result))]+ (is (= :result result))+ (is (= [[:set-autocommit false] :commit [:set-autocommit true] :close] @calls))))+ (testing "error path: rolls back, still resets autoCommit, and propagates the original exception"+ (let [calls (volatile! [])+ conn (recording-connection true calls)+ boom (ex-info "boom" {})]+ (is (identical? boom+ (try+ (#'mdb.connection/do-with-app-db-connection+ (mock-app-db :postgres conn)+ (fn [_] (throw boom)))+ (catch clojure.lang.ExceptionInfo e e))))+ (is (= [[:set-autocommit false] :rollback [:set-autocommit true] :close] @calls))))+ (testing "a connection that comes back already in manual-commit mode is taken over and reset (self-healing)"+ (let [calls (volatile! [])+ conn (recording-connection false calls)]+ (is (= :result+ (#'mdb.connection/do-with-app-db-connection+ (mock-app-db :postgres conn)+ (fn [_] :result))))+ (is (= [[:set-autocommit false] :commit [:set-autocommit true] :close] @calls))))))++(deftest do-with-app-db-connection-non-postgres-test+ (testing "when using a non-Postgres app DB, autoCommit is never touched -- the connection is just used and closed"+ (doseq [db-type [:h2 :mysql]]+ (let [calls (volatile! [])+ conn (recording-connection true calls)]+ (is (= :result+ (#'mdb.connection/do-with-app-db-connection+ (mock-app-db db-type conn)+ (fn [_] :result))))+ (is (= [:close] @calls) (str "db-type " db-type))))))