Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix OOM with sync metadata for postgres/redshift (#76218)

Alexander Polyankin4392d6aff40a
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.clj
index 10fa868640e..5bb5f9a20b5 100644
--- a/src/metabase/driver/sql_jdbc/execute.clj
+++ b/src/metabase/driver/sql_jdbc/execute.clj
@@ -50,8 +50,8 @@
;; whether this Connection should NOT be read-only, e.g. for DDL stuff or inserting data or whatever.
[:write? {:optional true, :default false} [:maybe :boolean]]
[:download? {:optional true} [:maybe :boolean]]
- ;; true if called from table-rows-sample-query
- [:sample? {:optional true} [:maybe :boolean]]
+ ;; true for large results we want to *stream* (a server-side cursor) rather than buffer in memory
+ [:stream? {:optional true} [:maybe :boolean]]
;; don't autoclose the connection
[:keep-open? {:optional true} [:maybe :boolean]]]])
@@ -396,17 +396,17 @@
;; `f`... we need to check and make sure that won't mess anything up, since some existing code is already doing it
;; manually. (metabase#40014)
(cond (not (or write?
- (and (-> options :download?) (= driver :postgres))))
+ (and (-> options :stream?) (isa? driver/hierarchy driver :postgres))))
(try
(log/trace (pr-str '(.setAutoCommit conn true)))
(.setAutoCommit conn true)
(catch Throwable e
(log/debug e "Error enabling connection autoCommit")))
- ;; todo (dan 7/11/25): fixing straightforward postgres oom on downloads in #60733, but seems like write? is
- ;; not set here. Note this is explicitly silent when `write?`. Lots of tests fail with autocommit false
- ;; there.
- (and (or (-> options :download?) (-> options :sample?)) (isa? driver/hierarchy driver :postgres))
+ ;; Postgres/Redshift buffer the *entire* ResultSet unless autoCommit is false (then they use a server-side
+ ;; cursor honoring the statement fetch size). So for streamed reads (`:stream?` -- sync metadata reads,
+ ;; table-rows-sample, downloads) flip autoCommit off, else a huge result OOMs.
+ (and (-> options :stream?) (isa? driver/hierarchy driver :postgres))
(try
(log/trace (pr-str '(.setAutoCommit conn false)))
(.setAutoCommit conn false)
@@ -802,7 +802,8 @@
(driver-api/database (driver-api/metadata-provider))
{:session-timezone (driver-api/report-timezone-id-if-supported driver (driver-api/database (driver-api/metadata-provider)))
:download? (download? (-> outer-query :info :context))
- :sample? (= :table-rows-sample (-> outer-query :info :context))}
+ :stream? (or (download? (-> outer-query :info :context))
+ (= :table-rows-sample (-> outer-query :info :context)))}
(fn [^Connection conn]
(with-open [stmt (statement-or-prepared-statement driver conn sql params (driver-api/canceled-chan))
^ResultSet rs (try
@@ -848,7 +849,7 @@
(do-with-connection-with-options
driver
db
- nil
+ {:stream? true}
(fn [^Connection conn]
(with-open [stmt (statement-or-prepared-statement driver conn sql params nil)
^ResultSet rs (try
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 5f538b0b06f..6183700d796 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -2528,3 +2528,15 @@
[(format "DROP USER IF EXISTS %s" qiso)])
(catch Throwable t
(log/warnf t "Test cleanup: DROP USER %s failed" qiso)))))))))))))
+
+(deftest ^:synchronized reducible-query-streams-large-result-set-test
+ (testing "reducible-query streams large result sets via a server-side cursor (autoCommit=false)"
+ (mt/test-driver *driver*
+ ;; A 2.1-billion-row generate_series in the SELECT list streams row-by-row (no server-side materialization).
+ ;; Pulling just the first few is fast ONLY if reducible-query streams (a cursor) and stops early; without
+ ;; streaming the JDBC driver buffers the whole ResultSet (~2.1B rows) and OOMs at any heap size.
+ (let [n Integer/MAX_VALUE
+ result (sql-jdbc.execute/reducible-query
+ (mt/db) [(format "SELECT generate_series(1, %d) AS i" n)])]
+ (testing "only the first rows are pulled, not all ~2 billion"
+ (is (= [1 2 3] (into [] (comp (take 3) (map :i)) result))))))))
diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.clj
index 96e9651f8e7..8affd02cde8 100644
--- a/test/metabase/driver/sql_jdbc/execute_test.clj
+++ b/test/metabase/driver/sql_jdbc/execute_test.clj
@@ -19,6 +19,7 @@
(me/humanize (mr/explain sql-jdbc.execute/ConnectionOptions options)))
nil nil
{} nil
+ {:stream? true} nil
{:session-timezone nil} nil
{:session-timezone "US/Pacific"} nil
{:session-timezone "X"} {:session-timezone ["invalid timezone ID: \"X\"" "timezone offset string literal"]}))