The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Reduce memory usage of metabase.sync on postgres (#72128)
Nathan Voxland1eebfd0c37d1
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.cljindex f652a90f699..db427f3ff92 100644--- a/src/metabase/app_db/core.clj+++ b/src/metabase/app_db/core.clj@@ -74,6 +74,7 @@qualifyqueryselect-or-insert!+ streaming-reduciblestreaming-reducible-querytype-keyword->descendantsupdate-or-insert!diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.cljindex d570a27bb84..91dd49d19a5 100644--- a/src/metabase/app_db/query.clj+++ b/src/metabase/app_db/query.clj@@ -147,19 +147,37 @@:uncompiled sql-args-or-honey-sql-map}e)))))))+(defn streaming-reducible+ "Returns a reducible that, when reduced, opens a transaction (autoCommit=false) with a+ fetch-size set to enable server-side cursor streaming on PostgreSQL app databases.++ `make-reducible-fn` is a 1-arg function called with the transaction `java.sql.Connection`.+ Pass it explicitly to `t2/reducible-select` via `:conn` or to `t2/reducible-query` so the+ query runs on the transaction connection — otherwise PostgreSQL ignores fetch-size.++ Example:+ (streaming-reducible+ (fn [conn]+ (eduction (map t2.realize/realize)+ (t2/reducible-select :conn conn :model/Table :active true))))"+ ([make-reducible-fn]+ (streaming-reducible make-reducible-fn 500))+ ([make-reducible-fn fetch-size]+ (reify clojure.lang.IReduceInit+ (reduce [_ f init]+ (t2/with-transaction [conn]+ (binding [t2.jdbc.options/*options* (merge t2.jdbc.options/*options*+ {:fetch-size fetch-size})]+ (reduce f init (make-reducible-fn conn))))))))+(defn streaming-reducible-query"Like `t2/reducible-query` but uses a transaction (autoCommit=false) and a fetch-size- to enable server-side cursor streaming on PostgreSQL. Returns a reducible."+ to enable server-side cursor streaming on PostgreSQL. Returns a reducible of raw maps."([sql-args-or-honey-sql-map](streaming-reducible-query sql-args-or-honey-sql-map 500))([sql-args-or-honey-sql-map fetch-size](let [compiled (compile sql-args-or-honey-sql-map)]- (reify clojure.lang.IReduceInit- (reduce [_ f init]- (t2/with-transaction [conn]- (binding [t2.jdbc.options/*options* (merge t2.jdbc.options/*options*- {:fetch-size fetch-size})]- (reduce f init (t2/reducible-query conn compiled)))))))))+ (streaming-reducible (fn [conn] (t2/reducible-query conn compiled)) fetch-size))))(defmacro with-conflict-retry"Retry a database mutation a single time if it fails due to concurrent insertions.diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.cljindex 0275b0c7d89..c526fbbd91d 100644--- a/src/metabase/sync/util.clj+++ b/src/metabase/sync/util.clj@@ -7,6 +7,7 @@[java-time.api :as t][medley.core :as m][metabase.analytics.prometheus :as prometheus]+ [metabase.app-db.core :as mdb][metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.events.core :as events]@@ -416,12 +417,14 @@(defn reducible-sync-tables"Returns a reducible of all the Tables that should go through the sync processes for `database-or-id`."[database-or-id & {:keys [schema-names table-names]}]- (eduction (map t2.realize/realize)- (t2/reducible-select :model/Table- :db_id (u/the-id database-or-id)- {:where [:and sync-tables-clause- (when (seq schema-names) [:in :schema schema-names])- (when (seq table-names) [:in :name table-names])]})))+ (mdb/streaming-reducible+ (fn [conn]+ (eduction (map t2.realize/realize)+ (t2/reducible-select :conn conn :model/Table+ :db_id (u/the-id database-or-id)+ {:where [:and sync-tables-clause+ (when (seq schema-names) [:in :schema schema-names])+ (when (seq table-names) [:in :name table-names])]})))))(defn sync-tables-count"The count of all tables that should be synced for `database-or-id`."@@ -432,17 +435,19 @@"A reducible collection of all the Tables that should go through the sync processes for `database-or-id`, in theorder they should be refingerprinted (by earliest last_analyzed timestamp)."[database-or-id]- (eduction (map t2.realize/realize)- (t2/reducible-select :model/Table- {:select [:t.*]- :from [[(t2/table-name :model/Table) :t]]- :left-join [[{:select [:table_id- [[:min :last_analyzed] :earliest_last_analyzed]]- :from [(t2/table-name :model/Field)]- :group-by [:table_id]} :sub]- [:= :t.id :sub.table_id]]- :where [:and sync-tables-clause [:= :t.db_id (u/the-id database-or-id)]]- :order-by [[:sub.earliest_last_analyzed :asc]]})))+ (mdb/streaming-reducible+ (fn [conn]+ (eduction (map t2.realize/realize)+ (t2/reducible-select :conn conn :model/Table+ {:select [:t.*]+ :from [[(t2/table-name :model/Table) :t]]+ :left-join [[{:select [:table_id+ [[:min :last_analyzed] :earliest_last_analyzed]]+ :from [(t2/table-name :model/Field)]+ :group-by [:table_id]} :sub]+ [:= :t.id :sub.table_id]]+ :where [:and sync-tables-clause [:= :t.db_id (u/the-id database-or-id)]]+ :order-by [[:sub.earliest_last_analyzed :asc]]})))))(defn sync-schemas"Returns all the Schemas that have their metadata sync'd for `database-or-id`.