The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Limit max sync / scan / fingerprint fields (#76189)
Alexander Polyankin6992729d5e1f
diff --git a/src/metabase/sync/analyze/fingerprint.clj b/src/metabase/sync/analyze/fingerprint.cljindex c3567e1852d..5515efa8002 100644--- a/src/metabase/sync/analyze/fingerprint.clj+++ b/src/metabase/sync/analyze/fingerprint.clj@@ -10,6 +10,7 @@[metabase.driver.util :as driver.u][metabase.lib.schema.metadata.fingerprint :as lib.schema.metadata.fingerprint][metabase.sync.interface :as i]+ [metabase.sync.settings :as sync.settings][metabase.sync.util :as sync-util][metabase.tracing.core :as tracing][metabase.util :as u]@@ -197,31 +198,53 @@;;; +----------------------------------------------------------------------------------------------------------------+(mu/defn- fields-to-fingerprint :- [:maybe [:sequential i/FieldInstance]]- "Return a sequences of Fields belonging to `table` for which we should generate (and save) fingerprints.- This should include NEW fields that are active and visible."- [table :- i/TableInstance]+ "Return up to `limit` Fields belonging to `table` for which we should generate (and save) fingerprints, ordered by+ id so the selection is stable across syncs. This should include NEW fields that are active and visible."+ [table :- i/TableInstance+ limit :- ms/PositiveInt](seq (t2/select :model/Field- (honeysql-for-fields-that-need-fingerprint-updating table))))+ (assoc (honeysql-for-fields-that-need-fingerprint-updating table)+ :order-by [[:id :asc]]+ :limit limit))))++(mu/defn- warn-too-many-fields!+ "Log that `table` has more fields to fingerprint than fingerprint-max-fields-per-table (`limit`), so only the first+ `limit` are fingerprinted and the rest skipped -- fingerprinting that many Fields at once can exhaust the heap (this+ has OOM'd instances syncing document databases like Mongo with very large/dynamic schemas)."+ [table :- i/TableInstance+ limit :- ms/PositiveInt]+ (log/warnf (str "Table %s has more than fingerprint-max-fields-per-table (%d) fields to fingerprint; fingerprinting "+ "the first %d and skipping the rest. Raise MB_FINGERPRINT_MAX_FIELDS_PER_TABLE to fingerprint more.")+ (sync-util/name-for-logging table) limit limit))++(mu/defn- fingerprint-fields-of-table!+ "Fingerprint the (non-empty) `fields` of `table`. On a non-transient error, advance their fingerprint version so we+ don't re-attempt every sync; transient errors are left untouched to retry next sync."+ [table :- i/TableInstance+ fields :- [:sequential i/FieldInstance]]+ (log/infof "Fingerprinting %s fields in table %s" (count fields) (sync-util/name-for-logging table))+ (let [stats (sync-util/with-returning-throwable (format "Error fingerprinting %s" (sync-util/name-for-logging table))+ (fingerprint-fields! table fields))]+ (if-let [throwable (:throwable stats)]+ (do+ (when-not (sync-util/transient-exception? throwable)+ (mark-fingerprinting-failed! fields))+ (merge (empty-stats-map 0) stats))+ stats)))(mu/defn fingerprint-table!- "Generate and save fingerprints for all the Fields in `table` that have not been previously analyzed."+ "Generate and save fingerprints for the Fields in `table` that have not been previously analyzed. At most+ [[metabase.sync.settings/fingerprint-max-fields-per-table]] fields are processed; if the table has more eligible+ fields, the rest are skipped (with a warning) so we don't load a huge number of Fields into memory at once."[table :- i/TableInstance](tracing/with-span :sync "sync.fingerprint.table" {:db/id (:db_id table) :sync/table (:name table)}- (if-let [fields (fields-to-fingerprint table)]- (do- (log/infof "Fingerprinting %s fields in table %s" (count fields) (sync-util/name-for-logging table))- (let [stats- (sync-util/with-returning-throwable (format "Error fingerprinting %s" (sync-util/name-for-logging table))- (fingerprint-fields! table fields))]- (if-let [throwable (:throwable stats)]- (do- ;; transient connection/environment errors abort this run and retry on the next sync; for any other- ;; (permanent) error, give up so we don't re-attempt every sync.- (when-not (sync-util/transient-exception? throwable)- (mark-fingerprinting-failed! fields))- (merge (empty-stats-map 0) stats))- stats)))- (empty-stats-map 0))))+ (let [limit (sync.settings/fingerprint-max-fields-per-table)+ fields (fields-to-fingerprint table (inc limit))]+ (when (> (count fields) limit)+ (warn-too-many-fields! table limit))+ (if-let [fields (seq (take limit fields))]+ (fingerprint-fields-of-table! table fields)+ (empty-stats-map 0)))))(def ^:private LogProgressFn[:=> [:cat :string [:schema i/TableInstance]] :any])diff --git a/src/metabase/sync/field_values.clj b/src/metabase/sync/field_values.cljindex 4044e050a00..cb4ab9f3ea7 100644--- a/src/metabase/sync/field_values.clj+++ b/src/metabase/sync/field_values.clj@@ -6,6 +6,7 @@[metabase.driver :as driver][metabase.driver.util :as driver.u][metabase.sync.interface :as i]+ [metabase.sync.settings :as sync.settings][metabase.sync.util :as sync-util][metabase.tracing.core :as tracing][metabase.util :as u]@@ -32,8 +33,22 @@(= ::field-values/fv-deleted result) {:deleted 1}))(defn- table->fields-to-scan- [table]- (t2/select :model/Field :table_id (u/the-id table), :active true, :visibility_type "normal"))+ "Up to `limit` active, normal-visibility Fields of `table`, ordered by id so the selection is stable across syncs.+ Bounding the count keeps wide tables (e.g. document databases with huge/dynamic schemas) from loading every Field+ into memory and issuing a warehouse request per field."+ [table limit]+ (t2/select :model/Field+ :table_id (u/the-id table), :active true, :visibility_type "normal"+ {:order-by [[:id :asc]] :limit limit}))++(defn- warn-too-many-fields!+ "Log that `table` has more fields to scan for FieldValues than scan-max-fields-per-table (`limit`), so only the first+ `limit` are scanned and the rest skipped -- scanning that many Fields would load them all into memory and (on non-SQL+ drivers like Mongo) issue a warehouse request per field."+ [table limit]+ (log/warnf (str "Table %s has more than scan-max-fields-per-table (%d) fields to scan for field values; scanning the "+ "first %d and skipping the rest. Raise MB_SCAN_MAX_FIELDS_PER_TABLE to scan more.")+ (sync-util/name-for-logging table) limit limit))(defn- can-batch-distinct?"Can this `table`'s database support the UNION ALL distinct-batch query? Requires:@@ -164,7 +179,11 @@SQL drivers use a single UNION ALL query covering every list-eligible field with an activeFieldValues row. Non-SQL drivers fall back to sequential per-field DISTINCT queries."[table :- i/TableInstance]- (let [all-fields (table->fields-to-scan table)+ (let [limit (sync.settings/scan-max-fields-per-table)+ scanned (table->fields-to-scan table (inc limit))+ _ (when (> (count scanned) limit)+ (warn-too-many-fields! table limit))+ all-fields (take limit scanned){to-sync trueto-clear false} (group-by #(boolean (field-values/field-should-have-field-values? %))all-fields)@@ -181,8 +200,8 @@(let [fv (get fvs-map (u/the-id field))](cond(not fv)- (do (log/infof "%s does not have FieldValues. Skipping..."- (sync-util/name-for-logging field))+ (do (log/tracef "%s does not have FieldValues. Skipping..."+ (sync-util/name-for-logging field))false)(field-values/inactive? fv)@@ -270,7 +289,7 @@For more info about advanced FieldValues, check the docsin [[metabase.warehouse-schema.models.field-values/field-values-types]]"[table :- i/TableInstance]- (->> (table->fields-to-scan table)+ (->> (table->fields-to-scan table (sync.settings/scan-max-fields-per-table))(map delete-expired-advanced-field-values-for-field!)(reduce +)))diff --git a/src/metabase/sync/settings.clj b/src/metabase/sync/settings.cljindex 9b529ed1c78..6269567995d 100644--- a/src/metabase/sync/settings.clj+++ b/src/metabase/sync/settings.clj@@ -42,3 +42,30 @@:export? true:default []:encryption :no)++(defsetting sync-max-fields-per-table+ "Maximum number of fields per table to sync as :model/Field rows. If a table's warehouse schema has more fields than+ this, only the first (by name) are synced and the rest are skipped -- keeps document databases with very large or+ dynamic schemas (e.g. MongoDB) from creating an unbounded number of Fields."+ :visibility :internal+ :export? true+ :type :integer+ :default 10000)++(defsetting scan-max-fields-per-table+ "Maximum number of fields per table to scan for field values. If a table has more active fields than this, the rest+ are skipped when scanning field values -- scanning that many fields would load them all into memory and, on non-SQL+ drivers like MongoDB, issue a warehouse request per field."+ :visibility :internal+ :export? true+ :type :integer+ :default 10000)++(defsetting fingerprint-max-fields-per-table+ "Maximum number of fields per table to fingerprint. If a table has more eligible fields than this, the rest are+ skipped during fingerprinting analysis -- loading that many fields into memory at once can exhaust the heap (this has+ OOM'd instances syncing document databases like MongoDB with very large or dynamic schemas)."+ :visibility :internal+ :export? true+ :type :integer+ :default 10000)diff --git a/src/metabase/sync/sync_metadata/fields.clj b/src/metabase/sync/sync_metadata/fields.cljindex 07e0281b4fa..eaddc2b099e 100644--- a/src/metabase/sync/sync_metadata/fields.clj+++ b/src/metabase/sync/sync_metadata/fields.clj@@ -42,6 +42,7 @@[metabase.driver.util :as driver.u][metabase.sync.fetch-metadata :as fetch-metadata][metabase.sync.interface :as i]+ [metabase.sync.settings :as sync.settings][metabase.sync.sync-metadata.fields.our-metadata :as fields.our-metadata][metabase.sync.sync-metadata.fields.sync-instances :as sync-instances][metabase.sync.sync-metadata.fields.sync-metadata :as sync-metadata]@@ -57,17 +58,42 @@;;; | PUTTING IT ALL TOGETHER |;;; +----------------------------------------------------------------------------------------------------------------++(mu/defn- warn-too-many-fields!+ "Log that `table`'s warehouse schema has more fields than sync-max-fields-per-table (`limit`), so only the first+ `limit` (by name) are synced as :model/Field rows and the rest are skipped -- keeps document databases with very+ large/dynamic schemas (e.g. MongoDB) from creating an unbounded number of Fields."+ [table :- i/TableInstance+ limit :- ms/PositiveInt]+ (log/warnf (str "Table %s has more than sync-max-fields-per-table (%d) fields; syncing the first %d (by name) and "+ "skipping the rest. Raise MB_SYNC_MAX_FIELDS_PER_TABLE to sync more.")+ (sync-util/name-for-logging table) limit limit))++(mu/defn- limit-fields-to-sync :- [:set i/TableMetadataField]+ "Cap `db-metadata` to at most [[sync.settings/sync-max-fields-per-table]] fields, keeping the first by name so the+ selection is stable across syncs, and warning when fields are dropped."+ [table :- i/TableInstance+ db-metadata :- [:set i/TableMetadataField]]+ (let [limit (sync.settings/sync-max-fields-per-table)]+ (if (> (count db-metadata) limit)+ (do+ (warn-too-many-fields! table limit)+ (into #{} (take limit (sort-by :name db-metadata))))+ db-metadata)))+(mu/defn- sync-and-update! :- ms/IntGreaterThanOrEqualToZero"Sync Field instances (i.e., rows in the Field table in the Metabase application DB) for a Table, and update metadata- properties (e.g. base type and comment/remark) as needed. Returns number of Fields synced."+ properties (e.g. base type and comment/remark) as needed. Returns number of Fields synced. Only the first+ [[sync.settings/sync-max-fields-per-table]] fields are synced; any beyond that are skipped (see+ [[limit-fields-to-sync]])."[database :- i/DatabaseInstancetable :- i/TableInstancedb-metadata :- [:set i/TableMetadataField]]- (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))- ;; Now that tables are synced and fields created as needed make sure field properties are in sync.- ;; Re-fetch our metadata because there might be some things that have changed after calling- ;; `sync-instances`- (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table))))+ (let [db-metadata (limit-fields-to-sync table db-metadata)]+ (+ (sync-instances/sync-instances! table db-metadata (fields.our-metadata/our-metadata table))+ ;; Now that tables are synced and fields created as needed make sure field properties are in sync.+ ;; Re-fetch our metadata because there might be some things that have changed after calling+ ;; `sync-instances`+ (sync-metadata/update-metadata! database table db-metadata (fields.our-metadata/our-metadata table)))))(defn- select-best-matching-name"Returns a key function for use with [[sort-by]] that ranks items based on how closely their `:schema` and `:name` match the given target values.@@ -80,7 +106,13 @@(mu/defn sync-fields! :- [:map[:updated-fields ms/IntGreaterThanOrEqualToZero][:total-fields ms/IntGreaterThanOrEqualToZero]]- "Sync the Fields in the Metabase application database for all the Tables in a `database`."+ "Sync the Fields in the Metabase application database for all the Tables in a `database`.++ `fields-metadata` is a *reducible* of per-column metadata, ordered so a table's columns are contiguous. It is passed+ straight to `sync!` (never `(into [] ...)`-ed): `sync!`'s `partition-by` buffers only one table's columns at a time,+ so memory stays bounded by the widest single table rather than the whole schema. A schema with a huge or churning+ set of tables (e.g. BigQuery with thousands of ephemeral tables) would otherwise materialize every column at once+ and OOM."[database :- i/DatabaseInstance](sync-util/with-error-handling (format "Error syncing Fields for Database ''%s''" (sync-util/name-for-logging database))(let [driver (driver.u/database->driver database)@@ -121,14 +153,13 @@(transduce (comp(map (fn [schema](log/infof "Fetching field metadata for %s.%s" (:name database) schema)- (into [] (fetch-metadata/fields-metadata database- :schema-names [schema]))))+ (fetch-metadata/fields-metadata database :schema-names [schema])))(map sync!))(completing (partial merge-with +)){:total-fields 0:updated-fields 0}(sync-util/sync-schemas database))- (sync! (into [] (fetch-metadata/fields-metadata database))))))))+ (sync! (fetch-metadata/fields-metadata database)))))))(mu/defn sync-fields-for-table!"Sync the Fields in the Metabase application database for a specific `table`."diff --git a/src/metabase/sync/sync_metadata/indexes.clj b/src/metabase/sync/sync_metadata/indexes.cljindex 8f39b17073f..ee8dfb43900 100644--- a/src/metabase/sync/sync_metadata/indexes.clj+++ b/src/metabase/sync/sync_metadata/indexes.clj@@ -49,32 +49,39 @@empty-stats))(defn- all-indexes->field-ids+ "Reduce the (reducible) whole-database `indexes` metadata to the set of indexed Field ids. Streams via a+ `partition-all` transducer so only one batch of index metadata is held in memory at a time, not every index in the+ database at once. Indexes are batched in groups of 5000 to stay under the 65,535 SQL parameter limit (see #52746)."[database-id indexes]- (reduce- (fn [accum index-batch]- (let [normal-indexes (map (juxt #(:table-schema % "__null__") :table-name :field-name) index-batch)- query (t2/reducible-query {:select [[:f.id]]- :from [[(t2/table-name :model/Field) :f]]- :inner-join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]]- :where [:and [:in [:composite [:coalesce :t.schema "__null__"] :t.name :f.name] normal-indexes]- [:= :t.db_id database-id]- [:= :parent_id nil]]})]- (into accum (keep :id) query)))+ (transduce+ (partition-all 5000)+ (completing+ (fn [accum index-batch]+ (let [normal-indexes (map (juxt #(:table-schema % "__null__") :table-name :field-name) index-batch)+ query (t2/reducible-query {:select [[:f.id]]+ :from [[(t2/table-name :model/Field) :f]]+ :inner-join [[(t2/table-name :model/Table) :t] [:= :f.table_id :t.id]]+ :where [:and [:in [:composite [:coalesce :t.schema "__null__"] :t.name :f.name] normal-indexes]+ [:= :t.db_id database-id]+ [:= :parent_id nil]]})]+ (into accum (keep :id) query))))#{}- ;; break the indexes up in groups of 5000 to avoid max- ;; parameter limit of 65,535. See #52746 for details.- (partition-all 5000 indexes)))+ indexes))(def ^:dynamic *update-partition-size*"Size of the partition of indexes to update using one `t2/update!` call. Dynamic for testing purposes."5000)(defn- sync-all-indexes!+ "Mark Fields in `database` as `database_indexed` based on the indexes reported by the driver. `describe-indexes` is a+ *reducible* over every index in the database; it is reduced straight through [[all-indexes->field-ids]] (never+ `(into [] ...)`-ed) so a database with a huge or churning set of tables doesn't materialize every index at once and+ OOM."[database](sync-util/with-error-handling "Error syncing Indexes"(let [indexes (fetch-metadata/log-if-error"index-metadata"- (into [] (driver/describe-indexes (driver.u/database->driver database) database)))+ (driver/describe-indexes (driver.u/database->driver database) database))database-id (:id database)indexed-field-ids (all-indexes->field-ids database-id indexes)existing-indexed-field-ids (t2/select-pks-set :model/Fielddiff --git a/test/metabase/sync/analyze/fingerprint_test.clj b/test/metabase/sync/analyze/fingerprint_test.cljindex bea3c88975c..23bf1b596e5 100644--- a/test/metabase/sync/analyze/fingerprint_test.clj+++ b/test/metabase/sync/analyze/fingerprint_test.clj@@ -166,6 +166,27 @@(def ^:private one-updated-map(merge default-stat-map {:updated-fingerprints 1, :fingerprints-attempted 1}))+(deftest fingerprint-table!-limits-fields-per-table-test+ (testing "fingerprints up to fingerprint-max-fields-per-table fields and skips the rest (avoids OOM on very wide tables)"+ (let [saved (atom 0)]+ (binding [i/*fingerprint-version->types-that-should-be-re-fingerprinted* {2 #{:type/Float}}]+ (mt/with-dynamic-fn-redefs [qp/process-query (fn [_query rff]+ (transduce identity (rff :metadata) [[1 1] [2 2] [3 3] [4 4] [5 5]]))+ sync.fingerprint/save-fingerprint! (fn [& _] (swap! saved inc))]+ (mt/with-temp [:model/Table table {}+ :model/Field _ {:table_id (u/the-id table) :base_type :type/Decimal :fingerprint_version 1 :active true}+ :model/Field _ {:table_id (u/the-id table) :base_type :type/Decimal :fingerprint_version 1 :active true}]+ (testing "limit of 1 -> only the first eligible field is fingerprinted, the rest skipped"+ (reset! saved 0)+ (mt/with-temporary-setting-values [fingerprint-max-fields-per-table 1]+ (sync.fingerprint/fingerprint-table! table)+ (is (= 1 @saved))))+ (testing "limit above the field count -> all eligible fields are fingerprinted"+ (reset! saved 0)+ (mt/with-temporary-setting-values [fingerprint-max-fields-per-table 10000]+ (sync.fingerprint/fingerprint-table! table)+ (is (= 2 @saved))))))))))+(deftest fingerprint-table!-test(testing "field is a substype of newer fingerprint version"(is (= [one-updated-map true]diff --git a/test/metabase/sync/field_values_test.clj b/test/metabase/sync/field_values_test.cljindex e962664e600..d37550b7e64 100644--- a/test/metabase/sync/field_values_test.clj+++ b/test/metabase/sync/field_values_test.clj@@ -307,6 +307,18 @@["update-field-values" {:throwable #(instance? Exception %)}]]}(sync/update-field-values! (data/db))))))))+;;; ---------------------------------- table->fields-to-scan ----------------------------------++(deftest table->fields-to-scan-respects-limit-test+ (testing "returns at most `limit` active, normal-visibility fields so very wide tables don't load every field"+ (mt/with-temp [:model/Table {table-id :id} {}+ :model/Field _ {:table_id table-id :active true :visibility_type :normal}+ :model/Field _ {:table_id table-id :active true :visibility_type :normal}+ :model/Field _ {:table_id table-id :active true :visibility_type :normal}]+ (let [table (t2/select-one :model/Table :id table-id)]+ (is (= 2 (count (#'sync.field-values/table->fields-to-scan table 2))))+ (is (= 3 (count (#'sync.field-values/table->fields-to-scan table 100))))))))+;;; ---------------------------------- can-batch-distinct? ----------------------------------(deftest can-batch-distinct?-testdiff --git a/test/metabase/sync/sync_metadata/fields_test.clj b/test/metabase/sync/sync_metadata/fields_test.cljindex 61ac75ece8d..86e71eecb2c 100644--- a/test/metabase/sync/sync_metadata/fields_test.clj+++ b/test/metabase/sync/sync_metadata/fields_test.clj@@ -73,6 +73,19 @@{:before-sync f-before:after-sync (f (mt/db))})))+(deftest limit-fields-to-sync-test+ (testing "caps the synced fields to sync-max-fields-per-table, keeping the first by name"+ (mt/with-temp [:model/Table table {}]+ (let [limit-fields (fn [db-metadata] (#'sync-fields/limit-fields-to-sync table db-metadata))+ field (fn [nm] {:name nm :database-type "varchar" :base-type :type/Text :database-position 0})+ three #{(field "c") (field "a") (field "b")}]+ (testing "over the limit -> only the first N by name are kept"+ (mt/with-temporary-setting-values [sync-max-fields-per-table 2]+ (is (= #{"a" "b"} (set (map :name (limit-fields three)))))))+ (testing "within the limit -> all kept unchanged"+ (mt/with-temporary-setting-values [sync-max-fields-per-table 100]+ (is (= three (limit-fields three)))))))))+(deftest renaming-fields-test(testing "make sure we can identify case changes on a field (#7923)"(let [db-state (with-test-db-before-and-after-altering