The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix OOM with sync metadata for postgres/redshift (#76218)
Alexander Polyankin4392d6aff40a
diff --git a/src/metabase/driver/sql_jdbc/execute.clj b/src/metabase/driver/sql_jdbc/execute.cljindex 10fa868640e..5bb5f9a20b5 100644--- a/src/metabase/driver/sql_jdbc/execute.clj+++ b/src/metabase/driver/sql_jdbc/execute.clj@@ -50,8 +50,8 @@;; whether this Connection should NOT be read-only, e.g. for DDL stuff or inserting data or whatever.[:write? {:optional true, :default false} [:maybe :boolean]][:download? {:optional true} [:maybe :boolean]]- ;; true if called from table-rows-sample-query- [:sample? {:optional true} [:maybe :boolean]]+ ;; true for large results we want to *stream* (a server-side cursor) rather than buffer in memory+ [:stream? {:optional true} [:maybe :boolean]];; don't autoclose the connection[:keep-open? {:optional true} [:maybe :boolean]]]])@@ -396,17 +396,17 @@;; `f`... we need to check and make sure that won't mess anything up, since some existing code is already doing it;; manually. (metabase#40014)(cond (not (or write?- (and (-> options :download?) (= driver :postgres))))+ (and (-> options :stream?) (isa? driver/hierarchy driver :postgres))))(try(log/trace (pr-str '(.setAutoCommit conn true)))(.setAutoCommit conn true)(catch Throwable e(log/debug e "Error enabling connection autoCommit")))- ;; todo (dan 7/11/25): fixing straightforward postgres oom on downloads in #60733, but seems like write? is- ;; not set here. Note this is explicitly silent when `write?`. Lots of tests fail with autocommit false- ;; there.- (and (or (-> options :download?) (-> options :sample?)) (isa? driver/hierarchy driver :postgres))+ ;; Postgres/Redshift buffer the *entire* ResultSet unless autoCommit is false (then they use a server-side+ ;; cursor honoring the statement fetch size). So for streamed reads (`:stream?` -- sync metadata reads,+ ;; table-rows-sample, downloads) flip autoCommit off, else a huge result OOMs.+ (and (-> options :stream?) (isa? driver/hierarchy driver :postgres))(try(log/trace (pr-str '(.setAutoCommit conn false)))(.setAutoCommit conn false)@@ -802,7 +802,8 @@(driver-api/database (driver-api/metadata-provider)){:session-timezone (driver-api/report-timezone-id-if-supported driver (driver-api/database (driver-api/metadata-provider))):download? (download? (-> outer-query :info :context))- :sample? (= :table-rows-sample (-> outer-query :info :context))}+ :stream? (or (download? (-> outer-query :info :context))+ (= :table-rows-sample (-> outer-query :info :context)))}(fn [^Connection conn](with-open [stmt (statement-or-prepared-statement driver conn sql params (driver-api/canceled-chan))^ResultSet rs (try@@ -848,7 +849,7 @@(do-with-connection-with-optionsdriverdb- nil+ {:stream? true}(fn [^Connection conn](with-open [stmt (statement-or-prepared-statement driver conn sql params nil)^ResultSet rs (trydiff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex 5f538b0b06f..6183700d796 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -2528,3 +2528,15 @@[(format "DROP USER IF EXISTS %s" qiso)])(catch Throwable t(log/warnf t "Test cleanup: DROP USER %s failed" qiso)))))))))))))++(deftest ^:synchronized reducible-query-streams-large-result-set-test+ (testing "reducible-query streams large result sets via a server-side cursor (autoCommit=false)"+ (mt/test-driver *driver*+ ;; A 2.1-billion-row generate_series in the SELECT list streams row-by-row (no server-side materialization).+ ;; Pulling just the first few is fast ONLY if reducible-query streams (a cursor) and stops early; without+ ;; streaming the JDBC driver buffers the whole ResultSet (~2.1B rows) and OOMs at any heap size.+ (let [n Integer/MAX_VALUE+ result (sql-jdbc.execute/reducible-query+ (mt/db) [(format "SELECT generate_series(1, %d) AS i" n)])]+ (testing "only the first rows are pulled, not all ~2 billion"+ (is (= [1 2 3] (into [] (comp (take 3) (map :i)) result))))))))diff --git a/test/metabase/driver/sql_jdbc/execute_test.clj b/test/metabase/driver/sql_jdbc/execute_test.cljindex 96e9651f8e7..8affd02cde8 100644--- a/test/metabase/driver/sql_jdbc/execute_test.clj+++ b/test/metabase/driver/sql_jdbc/execute_test.clj@@ -19,6 +19,7 @@(me/humanize (mr/explain sql-jdbc.execute/ConnectionOptions options)))nil nil{} nil+ {:stream? true} nil{:session-timezone nil} nil{:session-timezone "US/Pacific"} nil{:session-timezone "X"} {:session-timezone ["invalid timezone ID: \"X\"" "timezone offset string literal"]}))