Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Reduce memory usage of metabase.sync on postgres (#72128)

Nathan Voxland1eebfd0c37d1
diff --git a/src/metabase/app_db/core.clj b/src/metabase/app_db/core.clj
index f652a90f699..db427f3ff92 100644
--- a/src/metabase/app_db/core.clj
+++ b/src/metabase/app_db/core.clj
@@ -74,6 +74,7 @@
qualify
query
select-or-insert!
+ streaming-reducible
streaming-reducible-query
type-keyword->descendants
update-or-insert!
diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.clj
index d570a27bb84..91dd49d19a5 100644
--- a/src/metabase/app_db/query.clj
+++ b/src/metabase/app_db/query.clj
@@ -147,19 +147,37 @@
:uncompiled sql-args-or-honey-sql-map}
e)))))))
+(defn streaming-reducible
+ "Returns a reducible that, when reduced, opens a transaction (autoCommit=false) with a
+ fetch-size set to enable server-side cursor streaming on PostgreSQL app databases.
+
+ `make-reducible-fn` is a 1-arg function called with the transaction `java.sql.Connection`.
+ Pass it explicitly to `t2/reducible-select` via `:conn` or to `t2/reducible-query` so the
+ query runs on the transaction connection — otherwise PostgreSQL ignores fetch-size.
+
+ Example:
+ (streaming-reducible
+ (fn [conn]
+ (eduction (map t2.realize/realize)
+ (t2/reducible-select :conn conn :model/Table :active true))))"
+ ([make-reducible-fn]
+ (streaming-reducible make-reducible-fn 500))
+ ([make-reducible-fn fetch-size]
+ (reify clojure.lang.IReduceInit
+ (reduce [_ f init]
+ (t2/with-transaction [conn]
+ (binding [t2.jdbc.options/*options* (merge t2.jdbc.options/*options*
+ {:fetch-size fetch-size})]
+ (reduce f init (make-reducible-fn conn))))))))
+
(defn streaming-reducible-query
"Like `t2/reducible-query` but uses a transaction (autoCommit=false) and a fetch-size
- to enable server-side cursor streaming on PostgreSQL. Returns a reducible."
+ to enable server-side cursor streaming on PostgreSQL. Returns a reducible of raw maps."
([sql-args-or-honey-sql-map]
(streaming-reducible-query sql-args-or-honey-sql-map 500))
([sql-args-or-honey-sql-map fetch-size]
(let [compiled (compile sql-args-or-honey-sql-map)]
- (reify clojure.lang.IReduceInit
- (reduce [_ f init]
- (t2/with-transaction [conn]
- (binding [t2.jdbc.options/*options* (merge t2.jdbc.options/*options*
- {:fetch-size fetch-size})]
- (reduce f init (t2/reducible-query conn compiled)))))))))
+ (streaming-reducible (fn [conn] (t2/reducible-query conn compiled)) fetch-size))))
(defmacro with-conflict-retry
"Retry a database mutation a single time if it fails due to concurrent insertions.
diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.clj
index 0275b0c7d89..c526fbbd91d 100644
--- a/src/metabase/sync/util.clj
+++ b/src/metabase/sync/util.clj
@@ -7,6 +7,7 @@
[java-time.api :as t]
[medley.core :as m]
[metabase.analytics.prometheus :as prometheus]
+ [metabase.app-db.core :as mdb]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
@@ -416,12 +417,14 @@
(defn reducible-sync-tables
"Returns a reducible of all the Tables that should go through the sync processes for `database-or-id`."
[database-or-id & {:keys [schema-names table-names]}]
- (eduction (map t2.realize/realize)
- (t2/reducible-select :model/Table
- :db_id (u/the-id database-or-id)
- {:where [:and sync-tables-clause
- (when (seq schema-names) [:in :schema schema-names])
- (when (seq table-names) [:in :name table-names])]})))
+ (mdb/streaming-reducible
+ (fn [conn]
+ (eduction (map t2.realize/realize)
+ (t2/reducible-select :conn conn :model/Table
+ :db_id (u/the-id database-or-id)
+ {:where [:and sync-tables-clause
+ (when (seq schema-names) [:in :schema schema-names])
+ (when (seq table-names) [:in :name table-names])]})))))
(defn sync-tables-count
"The count of all tables that should be synced for `database-or-id`."
@@ -432,17 +435,19 @@
"A reducible collection of all the Tables that should go through the sync processes for `database-or-id`, in the
order they should be refingerprinted (by earliest last_analyzed timestamp)."
[database-or-id]
- (eduction (map t2.realize/realize)
- (t2/reducible-select :model/Table
- {:select [:t.*]
- :from [[(t2/table-name :model/Table) :t]]
- :left-join [[{:select [:table_id
- [[:min :last_analyzed] :earliest_last_analyzed]]
- :from [(t2/table-name :model/Field)]
- :group-by [:table_id]} :sub]
- [:= :t.id :sub.table_id]]
- :where [:and sync-tables-clause [:= :t.db_id (u/the-id database-or-id)]]
- :order-by [[:sub.earliest_last_analyzed :asc]]})))
+ (mdb/streaming-reducible
+ (fn [conn]
+ (eduction (map t2.realize/realize)
+ (t2/reducible-select :conn conn :model/Table
+ {:select [:t.*]
+ :from [[(t2/table-name :model/Table) :t]]
+ :left-join [[{:select [:table_id
+ [[:min :last_analyzed] :earliest_last_analyzed]]
+ :from [(t2/table-name :model/Field)]
+ :group-by [:table_id]} :sub]
+ [:= :t.id :sub.table_id]]
+ :where [:and sync-tables-clause [:= :t.db_id (u/the-id database-or-id)]]
+ :order-by [[:sub.earliest_last_analyzed :asc]]})))))
(defn sync-schemas
"Returns all the Schemas that have their metadata sync'd for `database-or-id`.