Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Push sync limit to drivers (#76307)

Alexander Polyankin5ec0d46be4db
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 8a592a4ad5a..95c52f3195a 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -272,11 +272,15 @@
(defn- dbfields->ftree*
"Construct _raw_ ftree from `dbfields`. Intended result is described in the [[dbfields->ftree]] docstring.
- _Raw_ means that nodes have only #{:database-type :visibility-type :index} keys."
- [dbfields]
+ _Raw_ means that nodes have only #{:database-type :visibility-type :index} keys.
+
+ Stops once the tree holds `limit` nodes (leaf + intermediate object fields), so collections with huge or deeply
+ nested schemas can't build an unbounded tree in memory (which has OOM'd sync). Nodes are counted by ftree path."
+ [dbfields limit]
(loop [[{raw-path :path leaf-type :type :as dbfield} & dbfields*] dbfields
- ftree {:children {}}]
- (if (nil? dbfield)
+ ftree {:children {}}
+ seen #{}]
+ (if (or (nil? dbfield) (>= (count seen) limit))
ftree
(let [ftree-paths (raw-path->ftree-paths raw-path)
parents-paths (butlast ftree-paths)
@@ -289,7 +293,7 @@
ftree* (reduce #(assoc-in %1 (conj (vec (first %2)) :index) (second %2))
ftree*
(map vector ftree-paths (:indices dbfield)))]
- (recur dbfields* ftree*)))))
+ (recur dbfields* ftree* (into seen ftree-paths))))))
(defn- ftree-prewalk
"Walk the `ftree` and apply (f ftree path) to each node prior descending to its children. Nodes are walked according
@@ -372,9 +376,11 @@
The resulting structure will hold at most [[driver.settings/sync-leaf-fields-limit]] leaf fields. That translates
to at most [[driver.settings/sync-leaf-fields-limit]] * [[describe-table-query-depth]]. That is 7K fields at the
- time of writing hence safe to reside in memory for further operation."
- [dbfields]
- (-> dbfields dbfields->ftree* ftree-reconcile-nodes))
+ time of writing hence safe to reside in memory for further operation. With `limit` it additionally caps the total
+ number of nodes (leaf + intermediate) -- a hard ceiling regardless of the leaf-field setting."
+ ([dbfields] (dbfields->ftree dbfields Long/MAX_VALUE))
+ ([dbfields limit]
+ (-> dbfields (dbfields->ftree* limit) ftree-reconcile-nodes)))
(defn- ftree->nested-fields
"Create nested-fields structure suitable for :fields key of structure return by [[driver/describe-table]]. `ftree`
@@ -428,7 +434,9 @@
[_driver database table]
{:schema nil
:name (:name table)
- :fields (-> (fetch-dbfields database table) dbfields->ftree ftree->nested-fields)})
+ :fields (-> (fetch-dbfields database table)
+ (dbfields->ftree (driver.settings/sync-max-fields-per-table))
+ ftree->nested-fields)})
;; describe-table impl end
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
index 91b45c0ea7f..adeb04667b0 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo_test.clj
@@ -306,6 +306,16 @@
:visibility-type :details-only}}}
(driver/describe-table :mongo (mt/db) (t2/select-one :model/Table :id (mt/id :nested-bindata)))))))))
+(deftest describe-table-respects-sync-max-fields-per-table-test
+ (mt/test-driver :mongo
+ (testing "describe-table caps the total number of fields at sync-max-fields-per-table"
+ ;; `venues` has 6 fields; with a tiny limit only that many are returned (capped during the tree build so a huge
+ ;; or deeply-nested collection can't OOM sync).
+ (mt/with-temporary-setting-values [sync-max-fields-per-table 2]
+ (is (= 2 (count (:fields (driver/describe-table
+ :mongo (mt/db)
+ (t2/select-one :model/Table :id (mt/id :venues)))))))))))
+
;; Index sync is turned off across the application as it is not used ATM.
#_(deftest sync-indexes-info-test
(mt/test-driver :mongo
diff --git a/src/metabase/driver/settings.clj b/src/metabase/driver/settings.clj
index 6a8ad505efd..a02305d7ffa 100644
--- a/src/metabase/driver/settings.clj
+++ b/src/metabase/driver/settings.clj
@@ -213,3 +213,12 @@
:export? true
:type :integer
:default 1000)
+
+(defsetting sync-max-fields-per-table
+ "Maximum number of fields per table to sync as :model/Field rows. If a table's warehouse schema has more fields than
+ this, only the first (by name) are synced and the rest are skipped -- keeps document databases with very large or
+ dynamic schemas (e.g. MongoDB) from creating an unbounded number of Fields."
+ :visibility :internal
+ :export? true
+ :type :integer
+ :default 10000)
diff --git a/src/metabase/driver/sql_jdbc/sync/describe_table.clj b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
index eb277a26b6d..241fbb0e2f7 100644
--- a/src/metabase/driver/sql_jdbc/sync/describe_table.clj
+++ b/src/metabase/driver/sql_jdbc/sync/describe_table.clj
@@ -365,8 +365,34 @@
[_driver _db & _args]
identity)
+(defn limit-fields-per-table-xf
+ "Stateful transducer that caps streamed `describe-fields` rows to at most `limit` per table -- a hard, O(1)-memory
+ per-table field cap (no buffering, no window functions). Rows must arrive contiguous by table, which the
+ `describe-fields` contract guarantees (ordered by `table-schema`, `table-name`); within a table the first `limit`
+ rows are kept and the rest dropped as they stream. [[describe-fields]] passes
+ [[metabase.driver.settings/sync-max-fields-per-table]] as `limit`, and the sync layer treats a table that comes back
+ with exactly that many fields as having hit the cap (see `metabase.sync.sync-metadata.fields/limit-fields-to-sync`)."
+ [limit]
+ (fn [rf]
+ (let [current (volatile! ::none)
+ n (volatile! 0)]
+ (fn
+ ([] (rf))
+ ([result] (rf result))
+ ([result row]
+ (let [table [(:table-schema row) (:table-name row)]]
+ (when-not (= table @current)
+ (vreset! current table)
+ (vreset! n 0))
+ (vswap! n inc)
+ (if (<= @n limit)
+ (rf result row)
+ result)))))))
+
(defn describe-fields
- "Default implementation of [[metabase.driver/describe-fields]] for JDBC drivers. Uses JDBC DatabaseMetaData."
+ "Default implementation of [[metabase.driver/describe-fields]] for JDBC drivers. Uses JDBC DatabaseMetaData. The
+ result is hard-capped to [[metabase.driver.settings/sync-max-fields-per-table]] fields per table (see
+ [[limit-fields-per-table-xf]])."
[driver db & {:keys [schema-names table-names] :as args}]
(if (or (and schema-names (empty? schema-names))
(and table-names (empty? table-names)))
@@ -382,6 +408,7 @@
(eduction
(comp
(m/mapply describe-fields-pre-process-xf driver db args)
+ (limit-fields-per-table-xf (driver.settings/sync-max-fields-per-table))
(describe-fields-xf driver db))
(sql-jdbc.execute/reducible-query db sql)))))
diff --git a/src/metabase/sync/settings.clj b/src/metabase/sync/settings.clj
index 6269567995d..daf184cf8f6 100644
--- a/src/metabase/sync/settings.clj
+++ b/src/metabase/sync/settings.clj
@@ -43,15 +43,6 @@
:default []
:encryption :no)
-(defsetting sync-max-fields-per-table
- "Maximum number of fields per table to sync as :model/Field rows. If a table's warehouse schema has more fields than
- this, only the first (by name) are synced and the rest are skipped -- keeps document databases with very large or
- dynamic schemas (e.g. MongoDB) from creating an unbounded number of Fields."
- :visibility :internal
- :export? true
- :type :integer
- :default 10000)
-
(defsetting scan-max-fields-per-table
"Maximum number of fields per table to scan for field values. If a table has more active fields than this, the rest
are skipped when scanning field values -- scanning that many fields would load them all into memory and, on non-SQL
diff --git a/src/metabase/sync/sync_metadata/fields.clj b/src/metabase/sync/sync_metadata/fields.clj
index eaddc2b099e..d351136f61b 100644
--- a/src/metabase/sync/sync_metadata/fields.clj
+++ b/src/metabase/sync/sync_metadata/fields.clj
@@ -39,10 +39,10 @@
* In general the methods in these namespaces return the number of rows updated; these numbers are summed and used
for logging purposes by higher-level sync logic."
(:require
+ [metabase.driver.settings :as driver.settings]
[metabase.driver.util :as driver.u]
[metabase.sync.fetch-metadata :as fetch-metadata]
[metabase.sync.interface :as i]
- [metabase.sync.settings :as sync.settings]
[metabase.sync.sync-metadata.fields.our-metadata :as fields.our-metadata]
[metabase.sync.sync-metadata.fields.sync-instances :as sync-instances]
[metabase.sync.sync-metadata.fields.sync-metadata :as sync-metadata]
@@ -69,21 +69,26 @@
(sync-util/name-for-logging table) limit limit))
(mu/defn- limit-fields-to-sync :- [:set i/TableMetadataField]
- "Cap `db-metadata` to at most [[sync.settings/sync-max-fields-per-table]] fields, keeping the first by name so the
- selection is stable across syncs, and warning when fields are dropped."
+ "Cap `db-metadata` to at most [[driver.settings/sync-max-fields-per-table]] fields, keeping the first by name so the
+ selection is stable across syncs, and warning when fields are dropped.
+
+ Drivers hard-cap their metadata at exactly `limit` fields per table (they read `sync-max-fields-per-table`), so a
+ table that comes back with `limit` fields is assumed to have hit the cap -- warn. We only need to truncate when a
+ driver returned strictly more than `limit` (i.e. one that doesn't honor the cap)."
[table :- i/TableInstance
db-metadata :- [:set i/TableMetadataField]]
- (let [limit (sync.settings/sync-max-fields-per-table)]
- (if (> (count db-metadata) limit)
- (do
- (warn-too-many-fields! table limit)
- (into #{} (take limit (sort-by :name db-metadata))))
+ (let [limit (driver.settings/sync-max-fields-per-table)
+ n (count db-metadata)]
+ (when (>= n limit)
+ (warn-too-many-fields! table limit))
+ (if (> n limit)
+ (into #{} (take limit (sort-by :name db-metadata)))
db-metadata)))
(mu/defn- sync-and-update! :- ms/IntGreaterThanOrEqualToZero
"Sync Field instances (i.e., rows in the Field table in the Metabase application DB) for a Table, and update metadata
properties (e.g. base type and comment/remark) as needed. Returns number of Fields synced. Only the first
- [[sync.settings/sync-max-fields-per-table]] fields are synced; any beyond that are skipped (see
+ [[driver.settings/sync-max-fields-per-table]] fields are synced; any beyond that are skipped (see
[[limit-fields-to-sync]])."
[database :- i/DatabaseInstance
table :- i/TableInstance