Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Do hold onto appdb transaction during database sync (#73246)

Nathan Voxland158f10882886
diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.clj
index 91dd49d19a5..797fc468cb5 100644
--- a/src/metabase/app_db/query.clj
+++ b/src/metabase/app_db/query.clj
@@ -155,6 +155,8 @@
Pass it explicitly to `t2/reducible-select` via `:conn` or to `t2/reducible-query` so the
query runs on the transaction connection — otherwise PostgreSQL ignores fetch-size.
+ WARNING: Because this opens a transaction, do not wrap any long-running logic in this which will hold the transaction open too long.
+
Example:
(streaming-reducible
(fn [conn]
diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.clj
index f59ed3fef30..e6240724c44 100644
--- a/src/metabase/sync/util.clj
+++ b/src/metabase/sync/util.clj
@@ -7,7 +7,6 @@
[java-time.api :as t]
[medley.core :as m]
[metabase.analytics-interface.core :as analytics]
- [metabase.app-db.core :as mdb]
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.events.core :as events]
@@ -417,14 +416,12 @@
(defn reducible-sync-tables
"Returns a reducible of all the Tables that should go through the sync processes for `database-or-id`."
[database-or-id & {:keys [schema-names table-names]}]
- (mdb/streaming-reducible
- (fn [conn]
- (eduction (map t2.realize/realize)
- (t2/reducible-select :conn conn :model/Table
- :db_id (u/the-id database-or-id)
- {:where [:and sync-tables-clause
- (when (seq schema-names) [:in :schema schema-names])
- (when (seq table-names) [:in :name table-names])]})))))
+ (eduction (map t2.realize/realize)
+ (t2/reducible-select :model/Table
+ :db_id (u/the-id database-or-id)
+ {:where [:and sync-tables-clause
+ (when (seq schema-names) [:in :schema schema-names])
+ (when (seq table-names) [:in :name table-names])]})))
(defn sync-tables-count
"The count of all tables that should be synced for `database-or-id`."
@@ -435,19 +432,17 @@
"A reducible collection of all the Tables that should go through the sync processes for `database-or-id`, in the
order they should be refingerprinted (by earliest last_analyzed timestamp)."
[database-or-id]
- (mdb/streaming-reducible
- (fn [conn]
- (eduction (map t2.realize/realize)
- (t2/reducible-select :conn conn :model/Table
- {:select [:t.*]
- :from [[(t2/table-name :model/Table) :t]]
- :left-join [[{:select [:table_id
- [[:min :last_analyzed] :earliest_last_analyzed]]
- :from [(t2/table-name :model/Field)]
- :group-by [:table_id]} :sub]
- [:= :t.id :sub.table_id]]
- :where [:and sync-tables-clause [:= :t.db_id (u/the-id database-or-id)]]
- :order-by [[:sub.earliest_last_analyzed :asc]]})))))
+ (eduction (map t2.realize/realize)
+ (t2/reducible-select :model/Table
+ {:select [:t.*]
+ :from [[(t2/table-name :model/Table) :t]]
+ :left-join [[{:select [:table_id
+ [[:min :last_analyzed] :earliest_last_analyzed]]
+ :from [(t2/table-name :model/Field)]
+ :group-by [:table_id]} :sub]
+ [:= :t.id :sub.table_id]]
+ :where [:and sync-tables-clause [:= :t.db_id (u/the-id database-or-id)]]
+ :order-by [[:sub.earliest_last_analyzed :asc]]})))
(defn sync-schemas
"Returns all the Schemas that have their metadata sync'd for `database-or-id`.