The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Bound the interestingness leftovers pass (#83964)
John Swanson2e651ffd1a25
diff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.cljindex f4e9ec2bfa9..7934697c34f 100644--- a/src/metabase/sync/analyze.clj+++ b/src/metabase/sync/analyze.clj@@ -91,9 +91,11 @@(format "Total number of tables classified %d, %d updated"total-tables tables-classified))-(defn- interestingness-summary [{:keys [fields-scored fields-failed]}]- (format "Interestingness scored %d fields, %d failed"- fields-scored fields-failed))+(defn- interestingness-summary [{:keys [fields-scored fields-failed fields-remaining]}]+ (str (format "Interestingness scored %d fields, %d failed" fields-scored fields-failed)+ ;; only present when the per-sync limit truncated the leftovers pass+ (when (and fields-remaining (pos? fields-remaining))+ (format ", %d still unscored" fields-remaining))))(defn- data-sensitivity-summary [{:keys [fields-scanned fields-labeled fields-failed]}](format "Data sensitivity scanned %d fields, labeled %d, %d failed"diff --git a/src/metabase/sync/analyze/interestingness.clj b/src/metabase/sync/analyze/interestingness.cljindex 7f6a098a8e2..708bf9786f4 100644--- a/src/metabase/sync/analyze/interestingness.clj+++ b/src/metabase/sync/analyze/interestingness.clj@@ -10,13 +10,15 @@outside the normal sync sweep, or scores null'ed to force a recompute)."(:require[metabase.interestingness.core :as interestingness]+ [metabase.lib.schema.id :as lib.schema.id][metabase.sync.db :as sync.db][metabase.sync.interface :as i]+ [metabase.sync.settings :as sync.settings][metabase.sync.util :as sync-util][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]- [toucan2.realize :as t2.realize]))+ [metabase.util.malli.registry :as mr]))(set! *warn-on-reflection* true)@@ -47,6 +49,13 @@fields)){:fields-scored 0 :fields-failed 0}))+(mr/def ::leftover-stats+ "Stats the leftovers pass accumulates. `:fields-remaining` appears only when the per-sync limit truncated the run."+ [:map {:closed true}+ [:fields-scored :int]+ [:fields-failed :int]+ [:fields-remaining {:optional true} :int]])+(defonce ^:private failed-leftover-field-ids;; Field IDs whose leftover scoring attempt failed earlier in this process. The leftovers pass;; selects on `dimension_interestingness IS NULL`, so without a marker a deterministically-failing@@ -55,26 +64,79 @@;; transient failures still get retried eventually.(atom #{}))-(mu/defn- score-missing-leftovers!+(def ^:private leftovers-group-size+ "How many Fields [[score-missing-leftovers!]] holds in memory at once. This is the pass's memory bound: whole Field+ rows carry a fingerprint each, so reading the whole NULL set at once exhausts the heap on a large database (a+ release that invalidates existing scores makes that set every field in the database). Not a setting -- it protects+ memory, which nobody needs to tune."+ 1000)++(mu/defn- score-group!+ "Score and persist the Fields with `field-ids`, skipping any that already failed in this process."+ [field-ids :- [:sequential {:min 1} ::lib.schema.id/field]+ stats :- ::leftover-stats]+ (let [failed @failed-leftover-field-ids]+ (reduce (fn [stats field]+ (let [result (score-and-save! field)]+ (if (instance? Exception result)+ (do+ (swap! failed-leftover-field-ids conj (u/the-id field))+ (update stats :fields-failed inc))+ (update stats :fields-scored inc))))+ stats+ (remove #(contains? failed (u/the-id %))+ (sync.db/fields-for-interestingness-scoring field-ids)))))++(defn- warn-skipped-failures!+ "Say so when Fields failed scoring earlier in this process and are being skipped."+ []+ (when-let [failed (not-empty @failed-leftover-field-ids)]+ (log/warnf (str "%d field(s) have failed interestingness scoring in this process and are being skipped;"+ " they keep no score until this process restarts.")+ (count failed))))++(mu/defn- truncated-stats :- ::leftover-stats+ "`stats` for a run the per-sync limit cut short, reporting how many Fields of `database` still have no score."+ [database :- i/DatabaseInstance+ budget :- :int+ stats :- ::leftover-stats]+ (let [remaining (sync.db/unscored-field-count-for-database (u/the-id database))]+ (log/warnf (str "Interestingness scoring stopped after %d fields for %s; %d still have no score and will be"+ " attempted by the next sync. Raise interestingness-max-fields-per-sync to converge sooner.")+ budget (sync-util/name-for-logging database) remaining)+ (assoc stats :fields-remaining remaining)))++(mu/defn- score-missing-leftovers! :- ::leftover-stats"Backup pass after the per-table sweep: any Field in `database` whose persisted`dimension_interestingness` is still `NULL` gets one more compute attempt. This catches Fieldson tables that aren't in `reducible-sync-tables` plus any fields the normal pipeline missed(initial backfill, prior compute failure, null'ed interestingness to force a recompute).Independent of fingerprint state; doesn't touch `last_analyzed`. Fields whose attempt already- failed in this process are skipped (see [[failed-leftover-field-ids]])."+ failed in this process are skipped (see [[failed-leftover-field-ids]]).++ Writes each group's scores before reading the next, so a run cut short keeps what it has already written. One run+ attempts at most [[sync.settings/interestingness-max-fields-per-sync]] Fields; whatever is left stays `NULL` and so+ is picked up by the next sync, and such a run reports `:fields-remaining`."[database :- i/DatabaseInstance]- (transduce (comp (remove #(contains? @failed-leftover-field-ids (u/the-id %)))- (map t2.realize/realize))- (completing- (fn [stats field]- (let [result (score-and-save! field)]- (if (instance? Exception result)- (do- (swap! failed-leftover-field-ids conj (u/the-id field))- (update stats :fields-failed inc))- (update stats :fields-scored inc)))))- {:fields-scored 0 :fields-failed 0}- (sync.db/unscored-fields-for-database-reducible (u/the-id database))))+ (let [database-id (u/the-id database)+ budget (sync.settings/interestingness-max-fields-per-sync)+ stats (loop [after-id 0+ attempted 0+ stats {:fields-scored 0 :fields-failed 0}]+ (let [room (- budget attempted)]+ (if-not (pos? room)+ (truncated-stats database budget stats)+ ;; ids arrive lowest first, so the cursor moves past every id this group looked at,+ ;; failures included -- a Field that cannot be scored costs one attempt per run rather+ ;; than stalling the pass on itself+ (if-let [field-ids (not-empty (sync.db/unscored-field-ids-for-database+ database-id after-id (min room leftovers-group-size)))]+ (recur (apply max field-ids)+ (+ attempted (count field-ids))+ (score-group! field-ids stats))+ stats))))]+ (warn-skipped-failures!)+ stats))(def ^:private LogProgressFn[:=> [:cat :string [:schema i/TableInstance]] :nil])diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.cljindex 31859d7f543..f64a7a0b989 100644--- a/src/metabase/sync/db.clj+++ b/src/metabase/sync/db.clj@@ -13,6 +13,7 @@[metabase.util :as u][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay][metabase.warehouse-schema.models.table :as table]@@ -387,18 +388,53 @@:semantic_type :type/Name{:from [(warehouse-schema-overlay/field-query)]}))-(mu/defn unscored-fields-for-database-reducible- "Reducible active, visible Fields of the Database with `database-id` without a dimension interestingness score."+(mr/def ::cursor+ "A keyset-pagination cursor over Field ids: zero starts before the first Field, since ids are positive."+ [:int {:min 0}])++(mu/defn- unscored-fields-clause+ "Honey SQL conditions matching the active, visible Fields of the Database with `database-id` that have no+ dimension interestingness score. Applied to [[warehouse-schema-overlay/field-query]], so `visibility_type` is the+ value as users see it."+ [database-id :- ::lib.schema.id/database]+ [:and+ [:= :active true]+ [:= :dimension_interestingness nil]+ [:not-in :visibility_type ["sensitive" "retired"]]+ [:in :table_id ^:allow-subquery {:select [:id]+ :from [(warehouse-schema-overlay/table-query {:user-settings? false})]+ :where [:= :db_id database-id]}]])++(mu/defn unscored-field-ids-for-database :- [:sequential ::lib.schema.id/field]+ "The ids of up to `limit` active, visible Fields of the Database with `database-id` without a dimension+ interestingness score, lowest id first, starting after `after-id`.++ Selecting ids alone keeps the page cheap enough to walk a very large `metabase_field` -- the ordering and the+ `after-id` bound are served by the primary key, and the caller fetches whole Fields a page at a time through+ [[fields-for-interestingness-scoring]]."+ [database-id :- ::lib.schema.id/database+ after-id :- ::cursor+ limit :- ms/PositiveInt]+ ;; `select-fn-vec` gives nil rather than an empty vector when nothing matches, which is every call once the+ ;; backfill has caught up+ (or (t2/select-fn-vec :id :model/Field+ {:select [:id]+ :from [(warehouse-schema-overlay/field-query)]+ :where [:and (unscored-fields-clause database-id) [:> :id after-id]]+ :order-by [[:id :asc]]+ :limit limit})+ []))++(mu/defn unscored-field-count-for-database :- :int+ "How many active, visible Fields of the Database with `database-id` have no dimension interestingness score."[database-id :- ::lib.schema.id/database]- (t2/reducible-select :model/Field- {:from [(warehouse-schema-overlay/field-query)]- :where [:and- [:= :active true]- [:= :dimension_interestingness nil]- [:not-in :visibility_type ["sensitive" "retired"]]- [:in :table_id ^:allow-subquery {:select [:id]- :from [(warehouse-schema-overlay/table-query {:user-settings? false})]- :where [:= :db_id database-id]}]]}))+ (t2/count :model/Field {:from [(warehouse-schema-overlay/field-query)]+ :where (unscored-fields-clause database-id)}))++(mu/defn fields-for-interestingness-scoring :- [:sequential (ms/InstanceOf :model/Field)]+ "The Fields with `field-ids`, as users see them, for interestingness scoring."+ [field-ids :- [:sequential {:min 1} ::lib.schema.id/field]]+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))(mu/defn top-level-field-ids-by-name"The IDs of the top-level Fields of the Table with `table-id` named one of `field-names`."diff --git a/src/metabase/sync/settings.clj b/src/metabase/sync/settings.cljindex fdec3a5030d..4c20ba7b4dd 100644--- a/src/metabase/sync/settings.clj+++ b/src/metabase/sync/settings.clj@@ -58,6 +58,17 @@:type :integer:default 10000)+(defsetting interestingness-max-fields-per-sync+ "Maximum number of fields per database to score for dimension interestingness in one sync. The leftovers pass+ scores every field whose score is still NULL, which after a release that invalidates existing scores is every field+ in the database; this bounds how long one sync spends on that backfill, leaving the rest for the next sync. Unlike+ `scan-max-fields-per-table` and `fingerprint-max-fields-per-table` the skipped fields are not abandoned -- they stay+ NULL and so are picked up by the following sync, which is why this can be set far higher than those two."+ :visibility :internal+ :export? true+ :type :integer+ :default 100000)+(defsetting data-sensitivity-scan-enabled"When true, the analyze phase of sync labels every unlabeled field with a data_sensitivity category inferred fromits name, types, and fingerprint, writing PUBLIC when nothing matches. Metadata only: the label does not mask ordiff --git a/test/metabase/sync/analyze/interestingness_test.clj b/test/metabase/sync/analyze/interestingness_test.cljindex 78c9dcb7a84..ee1bda7a893 100644--- a/test/metabase/sync/analyze/interestingness_test.clj+++ b/test/metabase/sync/analyze/interestingness_test.clj@@ -5,6 +5,7 @@[metabase.interestingness.dimension :as dim][metabase.sync.analyze.interestingness :as sync.interestingness][metabase.test :as mt]+ [metabase.util :as u][toucan2.core :as t2]));;; Smoke tests for the canonical weight profiles. The sync step itself is verified@@ -60,3 +61,101 @@(is (= {:fields-scored 0 :fields-failed 0}(#'sync.interestingness/score-missing-leftovers! database)))(is (= 1 @calls) "the failed field should be skipped, not re-scored on every sync"))))))++;;; ------------------------------------------- leftovers pass: paging and limits -------------------------------------++(defn- score-of [field]+ (t2/select-one-fn :dimension_interestingness :model/Field :id (u/the-id field)))++(defn- with-group-size! [n thunk]+ (with-redefs-fn {#'sync.interestingness/leftovers-group-size n} thunk))++(deftest score-missing-leftovers-pages-across-groups-test+ (testing "every unscored field is scored even when they outnumber what one group holds"+ (mt/with-temp [:model/Database database {}+ :model/Table table {:db_id (:id database)}+ :model/Field f1 {:table_id (:id table)}+ :model/Field f2 {:table_id (:id table)}+ :model/Field f3 {:table_id (:id table)}+ :model/Field f4 {:table_id (:id table)}+ :model/Field f5 {:table_id (:id table)}]+ (with-group-size! 2+ (fn []+ (is (= {:fields-scored 5 :fields-failed 0}+ (#'sync.interestingness/score-missing-leftovers! database))+ "a group size smaller than the field count must not stop the pass early")))+ (is (every? some? (map score-of [f1 f2 f3 f4 f5]))))))++(deftest score-missing-leftovers-stops-at-the-per-sync-limit-test+ (testing "one run scores at most the configured number of fields, and says how many are left"+ (mt/with-temp [:model/Database database {}+ :model/Table table {:db_id (:id database)}+ :model/Field _f1 {:table_id (:id table)}+ :model/Field _f2 {:table_id (:id table)}+ :model/Field _f3 {:table_id (:id table)}+ :model/Field _f4 {:table_id (:id table)}+ :model/Field _f5 {:table_id (:id table)}]+ (mt/with-temporary-setting-values [interestingness-max-fields-per-sync 2]+ (let [stats (#'sync.interestingness/score-missing-leftovers! database)]+ (is (= 2 (:fields-scored stats)))+ (is (= 0 (:fields-failed stats)))+ (is (= 3 (:fields-remaining stats))+ "a truncated run reports how many fields of this database are still unscored")))+ (testing "the scores written before the limit hit are committed, not rolled back"+ (is (= 3 (t2/count :model/Field :table_id (:id table) :dimension_interestingness nil))))+ (testing "the next run continues on the remainder rather than starting over"+ (mt/with-temporary-setting-values [interestingness-max-fields-per-sync 100]+ (is (= {:fields-scored 3 :fields-failed 0}+ (#'sync.interestingness/score-missing-leftovers! database)))))+ (testing "and a run with nothing left to do reports no remainder"+ (is (= {:fields-scored 0 :fields-failed 0}+ (#'sync.interestingness/score-missing-leftovers! database)))))))++(deftest score-missing-leftovers-keeps-going-past-a-failing-field-test+ (testing "a field whose scoring throws does not stop the rest of the run"+ (mt/with-temp [:model/Database database {}+ :model/Table table {:db_id (:id database)}+ :model/Field f1 {:table_id (:id table) :name "aaa"}+ :model/Field f2 {:table_id (:id table) :name "boom"}+ :model/Field f3 {:table_id (:id table) :name "zzz"}]+ (mt/with-dynamic-fn-redefs [interestingness/dimension-interestingness+ (fn [field]+ (if (= "boom" (:name field))+ (throw (ex-info "boom" {}))+ 0.5))]+ (with-group-size! 1+ (fn []+ (is (= {:fields-scored 2 :fields-failed 1}+ (#'sync.interestingness/score-missing-leftovers! database))+ "the failing field must not halt the pass, nor stall the cursor on itself"))))+ (is (= [0.5 nil 0.5] (map score-of [f1 f2 f3]))))))++(deftest score-missing-leftovers-persists-the-canonical-score-test+ (testing "the pass persists exactly what the scorer computes for the field"+ (let [fingerprint {:global {:distinct-count 5000 :nil% 0.0}+ :type {:type/DateTime {:earliest "2022-01-01" :latest "2024-12-31"}}}]+ (mt/with-temp [:model/Database database {}+ :model/Table table {:db_id (:id database)}+ :model/Field field {:table_id (:id table)+ :base_type :type/DateTime+ :semantic_type :type/CreationTimestamp+ :fingerprint fingerprint}]+ (is (= {:fields-scored 1 :fields-failed 0}+ (#'sync.interestingness/score-missing-leftovers! database)))+ (is (= (interestingness/dimension-interestingness+ (t2/select-one :model/Field :id (:id field)))+ (score-of field)))))))++(deftest score-missing-leftovers-respects-user-set-visibility-test+ (testing "a Field hidden by a user is not scored, as users' visibility_type is what the pass selects on"+ (mt/with-temp [:model/Database database {}+ :model/Table table {:db_id (:id database)}+ :model/Field visible {:table_id (:id table)}+ :model/Field hidden {:table_id (:id table)}]+ (t2/insert! :model/FieldUserSettings {:field_id (:id hidden)+ :visibility_type :sensitive})+ (is (= {:fields-scored 1 :fields-failed 0}+ (#'sync.interestingness/score-missing-leftovers! database)))+ (is (some? (score-of visible)))+ (is (nil? (score-of hidden))+ "the user-set visibility_type must still exclude the field after the rewrite"))))