Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Limit max sync / scan / fingerprint fields (#76189)

Alexander Polyankin6992729d5e1f
diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.clj
index c3567e1852d..5515efa8002 100644
--- a/src/metabase/sync/analyze/fingerprint.clj
+++ b/src/metabase/sync/analyze/fingerprint.clj
@@ -10,6 +10,7 @@
[metabase.driver.util :as driver.u]
[metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint]
[metabase.sync.interface :as i]
+ [metabase.sync.settings :as sync.settings]
[metabase.sync.util :as sync-util]
[metabase.tracing.core :as tracing]
[metabase.util :as u]
@@ -197,31 +198,53 @@
;;; +----------------------------------------------------------------------------------------------------------------+
(mu/defn- fields-to-fingerprint :- [:maybe [:sequential i/FieldInstance]]
- "Return a sequences of Fields belonging to `table` for which we should generate (and save) fingerprints.
- This should include NEW fields that are active and visible."
- [table :- i/TableInstance]
+ "Return up to `limit` Fields belonging to `table` for which we should generate (and save) fingerprints, ordered by
+ id so the selection is stable across syncs. This should include NEW fields that are active and visible."
+ [table :- i/TableInstance
+ limit :- ms/PositiveInt]
(seq (t2/select :model/Field
- (honeysql-for-fields-that-need-fingerprint-updating table))))
+ (assoc (honeysql-for-fields-that-need-fingerprint-updating table)
+ :order-by [[:id :asc]]
+ :limit limit))))
+
+(mu/defn- warn-too-many-fields!
+ "Log that `table` has more fields to fingerprint than fingerprint-max-fields-per-table (`limit`), so only the first
+ `limit` are fingerprinted and the rest skipped -- fingerprinting that many Fields at once can exhaust the heap (this
+ has OOM'd instances syncing document databases like Mongo with very large/dynamic schemas)."
+ [table :- i/TableInstance
+ limit :- ms/PositiveInt]
+ (log/warnf (str "Table %s has more than fingerprint-max-fields-per-table (%d) fields to fingerprint; fingerprinting "
+ "the first %d and skipping the rest. Raise MB_FINGERPRINT_MAX_FIELDS_PER_TABLE to fingerprint more.")
+ (sync-util/name-for-logging table) limit limit))
+
+(mu/defn- fingerprint-fields-of-table!
+ "Fingerprint the (non-empty) `fields` of `table`. On a non-transient error, advance their fingerprint version so we
+ don't re-attempt every sync; transient errors are left untouched to retry next sync."
+ [table :- i/TableInstance
+ fields :- [:sequential i/FieldInstance]]
+ (log/infof "Fingerprinting %s fields in table %s" (count fields) (sync-util/name-for-logging table))
+ (let [stats (sync-util/with-returning-throwable (format "Error fingerprinting %s" (sync-util/name-for-logging table))
+ (fingerprint-fields! table fields))]
+ (if-let [throwable (:throwable stats)]
+ (do
+ (when-not (sync-util/transient-exception? throwable)
+ (mark-fingerprinting-failed! fields))
+ (merge (empty-stats-map 0) stats))
+ stats)))
(mu/defn fingerprint-table!
- "Generate and save fingerprints for all the Fields in `table` that have not been previously analyzed."
+ "Generate and save fingerprints for the Fields in `table` that have not been previously analyzed. At most
+ [[metabase.sync.settings/fingerprint-max-fields-per-table]] fields are processed; if the table has more eligible
+ fields, the rest are skipped (with a warning) so we don't load a huge number of Fields into memory at once."
[table :- i/TableInstance]
(tracing/with-span :sync "sync.fingerprint.table" {:db/id (:db_id table) :sync/table (:name table)}
- (if-let [fields (fields-to-fingerprint table)]
- (do
- (log/infof "Fingerprinting %s fields in table %s" (count fields) (sync-util/name-for-logging table))
- (let [stats
- (sync-util/with-returning-throwable (format "Error fingerprinting %s" (sync-util/name-for-logging table))
- (fingerprint-fields! table fields))]
- (if-let [throwable (:throwable stats)]
- (do
- ;; transient connection/environment errors abort this run and retry on the next sync; for any other
- ;; (permanent) error, give up so we don't re-attempt every sync.
- (when-not (sync-util/transient-exception? throwable)
- (mark-fingerprinting-failed! fields))
- (merge (empty-stats-map 0) stats))
- stats)))
- (empty-stats-map 0))))
+ (let [limit (sync.settings/fingerprint-max-fields-per-table)
+ fields (fields-to-fingerprint table (inc limit))]
+ (when (> (count fields) limit)
+ (warn-too-many-fields! table limit))
+ (if-let [fields (seq (take limit fields))]
+ (fingerprint-fields-of-table! table fields)
+ (empty-stats-map 0)))))
(def ^:private LogProgressFn
[:=> [:cat :string [:schema i/TableInstance]] :any])
diff --git a/src/metabase/sync/field_values.clj b/src/metabase/sync/field_values.clj
index 4044e050a00..cb4ab9f3ea7 100644
--- a/src/metabase/sync/field_values.clj
+++ b/src/metabase/sync/field_values.clj
@@ -6,6 +6,7 @@
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
[metabase.sync.interface :as i]
+ [metabase.sync.settings :as sync.settings]
[metabase.sync.util :as sync-util]
[metabase.tracing.core :as tracing]
[metabase.util :as u]
@@ -32,8 +33,22 @@
(= ::field-values/fv-deleted result) {:deleted 1}))
(defn- table->fields-to-scan
- [table]
- (t2/select :model/Field :table_id (u/the-id table), :active true, :visibility_type "normal"))
+ "Up to `limit` active, normal-visibility Fields of `table`, ordered by id so the selection is stable across syncs.
+ Bounding the count keeps wide tables (e.g. document databases with huge/dynamic schemas) from loading every Field
+ into memory and issuing a warehouse request per field."
+ [table limit]
+ (t2/select :model/Field
+ :table_id (u/the-id table), :active true, :visibility_type "normal"
+ {:order-by [[:id :asc]] :limit limit}))
+
+(defn- warn-too-many-fields!
+ "Log that `table` has more fields to scan for FieldValues than scan-max-fields-per-table (`limit`), so only the first
+ `limit` are scanned and the rest skipped -- scanning that many Fields would load them all into memory and (on non-SQL
+ drivers like Mongo) issue a warehouse request per field."
+ [table limit]
+ (log/warnf (str "Table %s has more than scan-max-fields-per-table (%d) fields to scan for field values; scanning the "
+ "first %d and skipping the rest. Raise MB_SCAN_MAX_FIELDS_PER_TABLE to scan more.")
+ (sync-util/name-for-logging table) limit limit))
(defn- can-batch-distinct?
"Can this `table`'s database support the UNION ALL distinct-batch query? Requires:
@@ -164,7 +179,11 @@
SQL drivers use a single UNION ALL query covering every list-eligible field with an active
FieldValues row. Non-SQL drivers fall back to sequential per-field DISTINCT queries."
[table :- i/TableInstance]
- (let [all-fields (table->fields-to-scan table)
+ (let [limit (sync.settings/scan-max-fields-per-table)
+ scanned (table->fields-to-scan table (inc limit))
+ _ (when (> (count scanned) limit)
+ (warn-too-many-fields! table limit))
+ all-fields (take limit scanned)
{to-sync true
to-clear false} (group-by #(boolean (field-values/field-should-have-field-values? %))
all-fields)
@@ -181,8 +200,8 @@
(let [fv (get fvs-map (u/the-id field))]
(cond
(not fv)
- (do (log/infof "%s does not have FieldValues. Skipping..."
- (sync-util/name-for-logging field))
+ (do (log/tracef "%s does not have FieldValues. Skipping..."
+ (sync-util/name-for-logging field))
false)
(field-values/inactive? fv)
@@ -270,7 +289,7 @@
For more info about advanced FieldValues, check the docs
in [[metabase.warehouse-schema.models.field-values/field-values-types]]"
[table :- i/TableInstance]
- (->> (table->fields-to-scan table)
+ (->> (table->fields-to-scan table (sync.settings/scan-max-fields-per-table))
(map delete-expired-advanced-field-values-for-field!)
(reduce +)))
diff --git a/src/metabase/sync/settings.clj b/src/metabase/sync/settings.clj
index 9b529ed1c78..6269567995d 100644
--- a/src/metabase/sync/settings.clj
+++ b/src/metabase/sync/settings.clj
@@ -42,3 +42,30 @@
:export? true
:default []
:encryption :no)
+
+(defsetting sync-max-fields-per-table
+ "Maximum number of fields per table to sync as :model/Field rows. If a table's warehouse schema has more fields than
+ this, only the first (by name) are synced and the rest are skipped -- keeps document databases with very large or
+ dynamic schemas (e.g. MongoDB) from creating an unbounded number of Fields."
+ :visibility :internal
+ :export? true
+ :type :integer
+ :default 10000)
+
+(defsetting scan-max-fields-per-table
+ "Maximum number of fields per table to scan for field values. If a table has more active fields than this, the rest
+ are skipped when scanning field values -- scanning that many fields would load them all into memory and, on non-SQL
+ drivers like MongoDB, issue a warehouse request per field."
+ :visibility :internal
+ :export? true
+ :type :integer
+ :default 10000)
+
+(defsetting fingerprint-max-fields-per-table
+ "Maximum number of fields per table to fingerprint. If a table has more eligible fields than this, the rest are
+ skipped during fingerprinting analysis -- loading that many fields into memory at once can exhaust the heap (this has
+ OOM'd instances syncing document databases like MongoDB with very large or dynamic schemas)."
+ :visibility :internal
+ :export? true
+ :type :integer
+ :default 10000)
diff --git a/src/metabase/sync/sync_metadata/fields.clj b/src/metabase/sync/sync_metadata/fields.clj
index 07e0281b4fa..eaddc2b099e 100644
--- a/src/metabase/sync/sync_metadata/fields.clj
+++ b/src/metabase/sync/sync_metadata/fields.clj
@@ -42,6 +42,7 @@
[metabase.driver.util :as driver.u]
[metabase.sync.fetch-metadata :as fetch-metadata]
[metabase.sync.interface :as i]
+ [metabase.sync.settings :as sync.settings]
[metabase.sync.sync-metadata.fields.our-metadata :as fields.our-metadata]
[metabase.sync.sync-metadata.fields.sync-instances :as sync-instances]
[metabase.sync.sync-metadata.fields.sync-metadata :as sync-metadata]
@@ -57,17 +58,42 @@
;;; | PUTTING IT ALL TOGETHER |
;;; +----------------------------------------------------------------------------------------------------------------+
+(mu/defn- warn-too-many-fields!
+ "Log that `table`'s warehouse schema has more fields than sync-max-fields-per-table (`limit`), so only the first
+ `limit` (by name) are synced as :model/Field rows and the rest are skipped -- keeps document databases with very
+ large/dynamic schemas (e.g. MongoDB) from creating an unbounded number of Fields."
+ [table :- i/TableInstance
+ limit :- ms/PositiveInt]
+ (log/warnf (str "Table %s has more than sync-max-fields-per-table (%d) fields; syncing the first %d (by name) and "
+ "skipping the rest. Raise MB_SYNC_MAX_FIELDS_PER_TABLE to sync more.")
+ (sync-util/name-for-logging table) limit limit))
+
+(mu/defn- limit-fields-to-sync :- [:set i/TableMetadataField]
+ "Cap `db-metadata` to at most [[sync.settings/sync-max-fields-per-table]] fields, keeping the first by name so the
+ selection is stable across syncs, and warning when fields are dropped."
+ [table :- i/TableInstance
+ db-metadata :- [:set i/TableMetadataField]]
+ (let [limit (sync.settings/sync-max-fields-per-table)]
+ (if (> (count db-metadata) limit)
+ (do
+ (warn-too-many-fields! table limit)
+ (into #{} (take limit (sort-by :name db-metadata))))
+ db-metadata)))
+
(mu/defn- sync-and-update! :- ms/IntGreaterThanOrEqualToZero
"Sync Field instances (i.e., rows in the Field table in the Metabase application DB) for a Table, and update metadata
- properties (e.g. base type and comment/remark) as needed. Returns number of Fields synced."
+ properties (e.g. base type and comment/remark) as needed. Returns number of Fields synced. Only the first
+ [[sync.settings/sync-max-fields-per-table]] fields are synced; any beyond that are skipped (see
+ [[limit-fields-to-sync]])."
[database :- i/DatabaseInstance
table :- i/TableInstance
db-metadata :- [:set i/TableMetadataField]]
- (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))
- ;; Now that tables are synced and fields created as needed make sure field properties are in sync.
- ;; Re-fetch our metadata because there might be some things that have changed after calling
- ;; `sync-instances`
- (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table))))
+ (let [db-metadata (limit-fields-to-sync table db-metadata)]
+ (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))
+ ;; Now that tables are synced and fields created as needed make sure field properties are in sync.
+ ;; Re-fetch our metadata because there might be some things that have changed after calling
+ ;; `sync-instances`
+ (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table)))))
(defn- select-best-matching-name
"Returns a key function for use with [[sort-by]] that ranks items based on how closely their `:schema` and `:name` match the given target values.
@@ -80,7 +106,13 @@
(mu/defn sync-fields! :- [:map
[:updated-fields ms/IntGreaterThanOrEqualToZero]
[:total-fields ms/IntGreaterThanOrEqualToZero]]
- "Sync the Fields in the Metabase application database for all the Tables in a `database`."
+ "Sync the Fields in the Metabase application database for all the Tables in a `database`.
+
+ `fields-metadata` is a *reducible* of per-column metadata, ordered so a table's columns are contiguous. It is passed
+ straight to `sync!` (never `(into [] ...)`-ed): `sync!`'s `partition-by` buffers only one table's columns at a time,
+ so memory stays bounded by the widest single table rather than the whole schema. A schema with a huge or churning
+ set of tables (e.g. BigQuery with thousands of ephemeral tables) would otherwise materialize every column at once
+ and OOM."
[database :- i/DatabaseInstance]
(sync-util/with-error-handling (format "Error syncing Fields for Database ''%s''" (sync-util/name-for-logging database))
(let [driver (driver.u/database->driver database)
@@ -121,14 +153,13 @@
(transduce (comp
(map (fn [schema]
(log/infof "Fetching field metadata for %s.%s" (:name database) schema)
- (into [] (fetch-metadata/fields-metadata database
- :schema-names [schema]))))
+ (fetch-metadata/fields-metadata database :schema-names [schema])))
(map sync!))
(completing (partial merge-with +))
{:total-fields 0
:updated-fields 0}
(sync-util/sync-schemas database))
- (sync! (into [] (fetch-metadata/fields-metadata database))))))))
+ (sync! (fetch-metadata/fields-metadata database)))))))
(mu/defn sync-fields-for-table!
"Sync the Fields in the Metabase application database for a specific `table`."
diff --git a/src/metabase/sync/sync_metadata/indexes.clj b/src/metabase/sync/sync_metadata/indexes.clj
index 8f39b17073f..ee8dfb43900 100644
--- a/src/metabase/sync/sync_metadata/indexes.clj
+++ b/src/metabase/sync/sync_metadata/indexes.clj
@@ -49,32 +49,39 @@
empty-stats))
(defn- all-indexes->field-ids
+ "Reduce the (reducible) whole-database `indexes` metadata to the set of indexed Field ids. Streams via a
+ `partition-all` transducer so only one batch of index metadata is held in memory at a time, not every index in the
+ database at once. Indexes are batched in groups of 5000 to stay under the 65,535 SQL parameter limit (see #52746)."
[database-id indexes]
- (reduce
- (fn [accum index-batch]
- (let [normal-indexes (map (juxt #(:table-schema % "__null__") :table-name :field-name) index-batch)
- query (t2/reducible-query {:select [[:f.id]]
- :from [[(t2/table-name :model/Field) :f]]
- :inner-join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]]
- :where [:and [:in [:composite [:coalesce :t.schema "__null__"] :t.name :f.name] normal-indexes]
- [:= :t.db_id database-id]
- [:= :parent_id nil]]})]
- (into accum (keep :id) query)))
+ (transduce
+ (partition-all 5000)
+ (completing
+ (fn [accum index-batch]
+ (let [normal-indexes (map (juxt #(:table-schema % "__null__") :table-name :field-name) index-batch)
+ query (t2/reducible-query {:select [[:f.id]]
+ :from [[(t2/table-name :model/Field) :f]]
+ :inner-join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]]
+ :where [:and [:in [:composite [:coalesce :t.schema "__null__"] :t.name :f.name] normal-indexes]
+ [:= :t.db_id database-id]
+ [:= :parent_id nil]]})]
+ (into accum (keep :id) query))))
#{}
- ;; break the indexes up in groups of 5000 to avoid max
- ;; parameter limit of 65,535. See #52746 for details.
- (partition-all 5000 indexes)))
+ indexes))
(def ^:dynamic *update-partition-size*
"Size of the partition of indexes to update using one `t2/update!` call. Dynamic for testing purposes."
5000)
(defn- sync-all-indexes!
+ "Mark Fields in `database` as `database_indexed` based on the indexes reported by the driver. `describe-indexes` is a
+ *reducible* over every index in the database; it is reduced straight through [[all-indexes->field-ids]] (never
+ `(into [] ...)`-ed) so a database with a huge or churning set of tables doesn't materialize every index at once and
+ OOM."
[database]
(sync-util/with-error-handling "Error syncing Indexes"
(let [indexes (fetch-metadata/log-if-error
"index-metadata"
- (into [] (driver/describe-indexes (driver.u/database->driver database) database)))
+ (driver/describe-indexes (driver.u/database->driver database) database))
database-id (:id database)
indexed-field-ids (all-indexes->field-ids database-id indexes)
existing-indexed-field-ids (t2/select-pks-set :model/Field
diff --git a/test/metabase/sync/analyze/fingerprint_test.clj b/test/metabase/sync/analyze/fingerprint_test.clj
index bea3c88975c..23bf1b596e5 100644
--- a/test/metabase/sync/analyze/fingerprint_test.clj
+++ b/test/metabase/sync/analyze/fingerprint_test.clj
@@ -166,6 +166,27 @@
(def ^:private one-updated-map
(merge default-stat-map {:updated-fingerprints 1, :fingerprints-attempted 1}))
+(deftest fingerprint-table!-limits-fields-per-table-test
+ (testing "fingerprints up to fingerprint-max-fields-per-table fields and skips the rest (avoids OOM on very wide tables)"
+ (let [saved (atom 0)]
+ (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {2 #{:type/Float}}]
+ (mt/with-dynamic-fn-redefs [qp/process-query (fn [_query rff]
+ (transduce identity (rff :metadata) [[1 1] [2 2] [3 3] [4 4] [5 5]]))
+ sync.fingerprint/save-fingerprint! (fn [& _] (swap! saved inc))]
+ (mt/with-temp [:model/Table table {}
+ :model/Field _ {:table_id (u/the-id table) :base_type :type/Decimal :fingerprint_version 1 :active true}
+ :model/Field _ {:table_id (u/the-id table) :base_type :type/Decimal :fingerprint_version 1 :active true}]
+ (testing "limit of 1 -> only the first eligible field is fingerprinted, the rest skipped"
+ (reset! saved 0)
+ (mt/with-temporary-setting-values [fingerprint-max-fields-per-table 1]
+ (sync.fingerprint/fingerprint-table! table)
+ (is (= 1 @saved))))
+ (testing "limit above the field count -> all eligible fields are fingerprinted"
+ (reset! saved 0)
+ (mt/with-temporary-setting-values [fingerprint-max-fields-per-table 10000]
+ (sync.fingerprint/fingerprint-table! table)
+ (is (= 2 @saved))))))))))
+
(deftest fingerprint-table!-test
(testing "field is a substype of newer fingerprint version"
(is (= [one-updated-map true]
diff --git a/test/metabase/sync/field_values_test.clj b/test/metabase/sync/field_values_test.clj
index e962664e600..d37550b7e64 100644
--- a/test/metabase/sync/field_values_test.clj
+++ b/test/metabase/sync/field_values_test.clj
@@ -307,6 +307,18 @@
["update-field-values" {:throwable #(instance? Exception %)}]]}
(sync/update-field-values! (data/db))))))))
+;;; ---------------------------------- table->fields-to-scan ----------------------------------
+
+(deftest table->fields-to-scan-respects-limit-test
+ (testing "returns at most `limit` active, normal-visibility fields so very wide tables don't load every field"
+ (mt/with-temp [:model/Table {table-id :id} {}
+ :model/Field _ {:table_id table-id :active true :visibility_type :normal}
+ :model/Field _ {:table_id table-id :active true :visibility_type :normal}
+ :model/Field _ {:table_id table-id :active true :visibility_type :normal}]
+ (let [table (t2/select-one :model/Table :id table-id)]
+ (is (= 2 (count (#'sync.field-values/table->fields-to-scan table 2))))
+ (is (= 3 (count (#'sync.field-values/table->fields-to-scan table 100))))))))
+
;;; ---------------------------------- can-batch-distinct? ----------------------------------
(deftest can-batch-distinct?-test
diff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.clj
index 61ac75ece8d..86e71eecb2c 100644
--- a/test/metabase/sync/sync_metadata/fields_test.clj
+++ b/test/metabase/sync/sync_metadata/fields_test.clj
@@ -73,6 +73,19 @@
{:before-sync f-before
:after-sync (f (mt/db))})))
+(deftest limit-fields-to-sync-test
+ (testing "caps the synced fields to sync-max-fields-per-table, keeping the first by name"
+ (mt/with-temp [:model/Table table {}]
+ (let [limit-fields (fn [db-metadata] (#'sync-fields/limit-fields-to-sync table db-metadata))
+ field (fn [nm] {:name nm :database-type "varchar" :base-type :type/Text :database-position 0})
+ three #{(field "c") (field "a") (field "b")}]
+ (testing "over the limit -> only the first N by name are kept"
+ (mt/with-temporary-setting-values [sync-max-fields-per-table 2]
+ (is (= #{"a" "b"} (set (map :name (limit-fields three)))))))
+ (testing "within the limit -> all kept unchanged"
+ (mt/with-temporary-setting-values [sync-max-fields-per-table 100]
+ (is (= three (limit-fields three)))))))))
+
(deftest renaming-fields-test
(testing "make sure we can identify case changes on a field (#7923)"
(let [db-state (with-test-db-before-and-after-altering