Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Bound the interestingness leftovers pass (#83964)

John Swanson2e651ffd1a25
diff --git a/src/metabase/sync/analyze.clj b/src/metabase/sync/analyze.clj
index f4e9ec2bfa9..7934697c34f 100644
--- a/src/metabase/sync/analyze.clj
+++ b/src/metabase/sync/analyze.clj
@@ -91,9 +91,11 @@
(format "Total number of tables classified %d, %d updated"
total-tables tables-classified))
-(defn- interestingness-summary [{:keys [fields-scored fields-failed]}]
- (format "Interestingness scored %d fields, %d failed"
- fields-scored fields-failed))
+(defn- interestingness-summary [{:keys [fields-scored fields-failed fields-remaining]}]
+ (str (format "Interestingness scored %d fields, %d failed" fields-scored fields-failed)
+ ;; only present when the per-sync limit truncated the leftovers pass
+ (when (and fields-remaining (pos? fields-remaining))
+ (format ", %d still unscored" fields-remaining))))
(defn- data-sensitivity-summary [{:keys [fields-scanned fields-labeled fields-failed]}]
(format "Data sensitivity scanned %d fields, labeled %d, %d failed"
diff --git a/src/metabase/sync/analyze/interestingness.clj b/src/metabase/sync/analyze/interestingness.clj
index 7f6a098a8e2..708bf9786f4 100644
--- a/src/metabase/sync/analyze/interestingness.clj
+++ b/src/metabase/sync/analyze/interestingness.clj
@@ -10,13 +10,15 @@
outside the normal sync sweep, or scores null'ed to force a recompute)."
(:require
[metabase.interestingness.core :as interestingness]
+ [metabase.lib.schema.id :as lib.schema.id]
[metabase.sync.db :as sync.db]
[metabase.sync.interface :as i]
+ [metabase.sync.settings :as sync.settings]
[metabase.sync.util :as sync-util]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [toucan2.realize :as t2.realize]))
+ [metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
@@ -47,6 +49,13 @@
fields))
{:fields-scored 0 :fields-failed 0}))
+(mr/def ::leftover-stats
+ "Stats the leftovers pass accumulates. `:fields-remaining` appears only when the per-sync limit truncated the run."
+ [:map {:closed true}
+ [:fields-scored :int]
+ [:fields-failed :int]
+ [:fields-remaining {:optional true} :int]])
+
(defonce ^:private failed-leftover-field-ids
;; Field IDs whose leftover scoring attempt failed earlier in this process. The leftovers pass
;; selects on `dimension_interestingness IS NULL`, so without a marker a deterministically-failing
@@ -55,26 +64,79 @@
;; transient failures still get retried eventually.
(atom #{}))
-(mu/defn- score-missing-leftovers!
+(def ^:private leftovers-group-size
+ "How many Fields [[score-missing-leftovers!]] holds in memory at once. This is the pass's memory bound: whole Field
+ rows carry a fingerprint each, so reading the whole NULL set at once exhausts the heap on a large database (a
+ release that invalidates existing scores makes that set every field in the database). Not a setting -- it protects
+ memory, which nobody needs to tune."
+ 1000)
+
+(mu/defn- score-group!
+ "Score and persist the Fields with `field-ids`, skipping any that already failed in this process."
+ [field-ids :- [:sequential {:min 1} ::lib.schema.id/field]
+ stats :- ::leftover-stats]
+ (let [failed @failed-leftover-field-ids]
+ (reduce (fn [stats field]
+ (let [result (score-and-save! field)]
+ (if (instance? Exception result)
+ (do
+ (swap! failed-leftover-field-ids conj (u/the-id field))
+ (update stats :fields-failed inc))
+ (update stats :fields-scored inc))))
+ stats
+ (remove #(contains? failed (u/the-id %))
+ (sync.db/fields-for-interestingness-scoring field-ids)))))
+
+(defn- warn-skipped-failures!
+ "Say so when Fields failed scoring earlier in this process and are being skipped."
+ []
+ (when-let [failed (not-empty @failed-leftover-field-ids)]
+ (log/warnf (str "%d field(s) have failed interestingness scoring in this process and are being skipped;"
+ " they keep no score until this process restarts.")
+ (count failed))))
+
+(mu/defn- truncated-stats :- ::leftover-stats
+ "`stats` for a run the per-sync limit cut short, reporting how many Fields of `database` still have no score."
+ [database :- i/DatabaseInstance
+ budget :- :int
+ stats :- ::leftover-stats]
+ (let [remaining (sync.db/unscored-field-count-for-database (u/the-id database))]
+ (log/warnf (str "Interestingness scoring stopped after %d fields for %s; %d still have no score and will be"
+ " attempted by the next sync. Raise interestingness-max-fields-per-sync to converge sooner.")
+ budget (sync-util/name-for-logging database) remaining)
+ (assoc stats :fields-remaining remaining)))
+
+(mu/defn- score-missing-leftovers! :- ::leftover-stats
"Backup pass after the per-table sweep: any Field in `database` whose persisted
`dimension_interestingness` is still `NULL` gets one more compute attempt. This catches Fields
on tables that aren't in `reducible-sync-tables` plus any fields the normal pipeline missed
(initial backfill, prior compute failure, null'ed interestingness to force a recompute).
Independent of fingerprint state; doesn't touch `last_analyzed`. Fields whose attempt already
- failed in this process are skipped (see [[failed-leftover-field-ids]])."
+ failed in this process are skipped (see [[failed-leftover-field-ids]]).
+
+ Writes each group's scores before reading the next, so a run cut short keeps what it has already written. One run
+ attempts at most [[sync.settings/interestingness-max-fields-per-sync]] Fields; whatever is left stays `NULL` and so
+ is picked up by the next sync, and such a run reports `:fields-remaining`."
[database :- i/DatabaseInstance]
- (transduce (comp (remove #(contains? @failed-leftover-field-ids (u/the-id %)))
- (map t2.realize/realize))
- (completing
- (fn [stats field]
- (let [result (score-and-save! field)]
- (if (instance? Exception result)
- (do
- (swap! failed-leftover-field-ids conj (u/the-id field))
- (update stats :fields-failed inc))
- (update stats :fields-scored inc)))))
- {:fields-scored 0 :fields-failed 0}
- (sync.db/unscored-fields-for-database-reducible (u/the-id database))))
+ (let [database-id (u/the-id database)
+ budget (sync.settings/interestingness-max-fields-per-sync)
+ stats (loop [after-id 0
+ attempted 0
+ stats {:fields-scored 0 :fields-failed 0}]
+ (let [room (- budget attempted)]
+ (if-not (pos? room)
+ (truncated-stats database budget stats)
+ ;; ids arrive lowest first, so the cursor moves past every id this group looked at,
+ ;; failures included -- a Field that cannot be scored costs one attempt per run rather
+ ;; than stalling the pass on itself
+ (if-let [field-ids (not-empty (sync.db/unscored-field-ids-for-database
+ database-id after-id (min room leftovers-group-size)))]
+ (recur (apply max field-ids)
+ (+ attempted (count field-ids))
+ (score-group! field-ids stats))
+ stats))))]
+ (warn-skipped-failures!)
+ stats))
(def ^:private LogProgressFn
[:=> [:cat :string [:schema i/TableInstance]] :nil])
diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.clj
index 31859d7f543..f64a7a0b989 100644
--- a/src/metabase/sync/db.clj
+++ b/src/metabase/sync/db.clj
@@ -13,6 +13,7 @@
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[metabase.warehouse-schema-overlay.core :as warehouse-schema-overlay]
[metabase.warehouse-schema.models.table :as table]
@@ -387,18 +388,53 @@
:semantic_type :type/Name
{:from [(warehouse-schema-overlay/field-query)]}))
-(mu/defn unscored-fields-for-database-reducible
- "Reducible active, visible Fields of the Database with `database-id` without a dimension interestingness score."
+(mr/def ::cursor
+ "A keyset-pagination cursor over Field ids: zero starts before the first Field, since ids are positive."
+ [:int {:min 0}])
+
+(mu/defn- unscored-fields-clause
+ "Honey SQL conditions matching the active, visible Fields of the Database with `database-id` that have no
+ dimension interestingness score. Applied to [[warehouse-schema-overlay/field-query]], so `visibility_type` is the
+ value as users see it."
+ [database-id :- ::lib.schema.id/database]
+ [:and
+ [:= :active true]
+ [:= :dimension_interestingness nil]
+ [:not-in :visibility_type ["sensitive" "retired"]]
+ [:in :table_id ^:allow-subquery {:select [:id]
+ :from [(warehouse-schema-overlay/table-query {:user-settings? false})]
+ :where [:= :db_id database-id]}]])
+
+(mu/defn unscored-field-ids-for-database :- [:sequential ::lib.schema.id/field]
+ "The ids of up to `limit` active, visible Fields of the Database with `database-id` without a dimension
+ interestingness score, lowest id first, starting after `after-id`.
+
+ Selecting ids alone keeps the page cheap enough to walk a very large `metabase_field` -- the ordering and the
+ `after-id` bound are served by the primary key, and the caller fetches whole Fields a page at a time through
+ [[fields-for-interestingness-scoring]]."
+ [database-id :- ::lib.schema.id/database
+ after-id :- ::cursor
+ limit :- ms/PositiveInt]
+ ;; `select-fn-vec` gives nil rather than an empty vector when nothing matches, which is every call once the
+ ;; backfill has caught up
+ (or (t2/select-fn-vec :id :model/Field
+ {:select [:id]
+ :from [(warehouse-schema-overlay/field-query)]
+ :where [:and (unscored-fields-clause database-id) [:> :id after-id]]
+ :order-by [[:id :asc]]
+ :limit limit})
+ []))
+
+(mu/defn unscored-field-count-for-database :- :int
+ "How many active, visible Fields of the Database with `database-id` have no dimension interestingness score."
[database-id :- ::lib.schema.id/database]
- (t2/reducible-select :model/Field
- {:from [(warehouse-schema-overlay/field-query)]
- :where [:and
- [:= :active true]
- [:= :dimension_interestingness nil]
- [:not-in :visibility_type ["sensitive" "retired"]]
- [:in :table_id ^:allow-subquery {:select [:id]
- :from [(warehouse-schema-overlay/table-query {:user-settings? false})]
- :where [:= :db_id database-id]}]]}))
+ (t2/count :model/Field {:from [(warehouse-schema-overlay/field-query)]
+ :where (unscored-fields-clause database-id)}))
+
+(mu/defn fields-for-interestingness-scoring :- [:sequential (ms/InstanceOf :model/Field)]
+ "The Fields with `field-ids`, as users see them, for interestingness scoring."
+ [field-ids :- [:sequential {:min 1} ::lib.schema.id/field]]
+ (t2/select :model/Field :id [:in field-ids] {:from [(warehouse-schema-overlay/field-query)]}))
(mu/defn top-level-field-ids-by-name
"The IDs of the top-level Fields of the Table with `table-id` named one of `field-names`."
diff --git a/src/metabase/sync/settings.clj b/src/metabase/sync/settings.clj
index fdec3a5030d..4c20ba7b4dd 100644
--- a/src/metabase/sync/settings.clj
+++ b/src/metabase/sync/settings.clj
@@ -58,6 +58,17 @@
:type :integer
:default 10000)
+(defsetting interestingness-max-fields-per-sync
+ "Maximum number of fields per database to score for dimension interestingness in one sync. The leftovers pass
+ scores every field whose score is still NULL, which after a release that invalidates existing scores is every field
+ in the database; this bounds how long one sync spends on that backfill, leaving the rest for the next sync. Unlike
+ `scan-max-fields-per-table` and `fingerprint-max-fields-per-table` the skipped fields are not abandoned -- they stay
+ NULL and so are picked up by the following sync, which is why this can be set far higher than those two."
+ :visibility :internal
+ :export? true
+ :type :integer
+ :default 100000)
+
(defsetting data-sensitivity-scan-enabled
"When true, the analyze phase of sync labels every unlabeled field with a data_sensitivity category inferred from
its name, types, and fingerprint, writing PUBLIC when nothing matches. Metadata only: the label does not mask or
diff --git a/test/metabase/sync/analyze/interestingness_test.clj b/test/metabase/sync/analyze/interestingness_test.clj
index 78c9dcb7a84..ee1bda7a893 100644
--- a/test/metabase/sync/analyze/interestingness_test.clj
+++ b/test/metabase/sync/analyze/interestingness_test.clj
@@ -5,6 +5,7 @@
[metabase.interestingness.dimension :as dim]
[metabase.sync.analyze.interestingness :as sync.interestingness]
[metabase.test :as mt]
+ [metabase.util :as u]
[toucan2.core :as t2]))
;;; Smoke tests for the canonical weight profiles. The sync step itself is verified
@@ -60,3 +61,101 @@
(is (= {:fields-scored 0 :fields-failed 0}
(#'sync.interestingness/score-missing-leftovers! database)))
(is (= 1 @calls) "the failed field should be skipped, not re-scored on every sync"))))))
+
+;;; ------------------------------------------- leftovers pass: paging and limits -------------------------------------
+
+(defn- score-of [field]
+ (t2/select-one-fn :dimension_interestingness :model/Field :id (u/the-id field)))
+
+(defn- with-group-size! [n thunk]
+ (with-redefs-fn {#'sync.interestingness/leftovers-group-size n} thunk))
+
+(deftest score-missing-leftovers-pages-across-groups-test
+ (testing "every unscored field is scored even when they outnumber what one group holds"
+ (mt/with-temp [:model/Database database {}
+ :model/Table table {:db_id (:id database)}
+ :model/Field f1 {:table_id (:id table)}
+ :model/Field f2 {:table_id (:id table)}
+ :model/Field f3 {:table_id (:id table)}
+ :model/Field f4 {:table_id (:id table)}
+ :model/Field f5 {:table_id (:id table)}]
+ (with-group-size! 2
+ (fn []
+ (is (= {:fields-scored 5 :fields-failed 0}
+ (#'sync.interestingness/score-missing-leftovers! database))
+ "a group size smaller than the field count must not stop the pass early")))
+ (is (every? some? (map score-of [f1 f2 f3 f4 f5]))))))
+
+(deftest score-missing-leftovers-stops-at-the-per-sync-limit-test
+ (testing "one run scores at most the configured number of fields, and says how many are left"
+ (mt/with-temp [:model/Database database {}
+ :model/Table table {:db_id (:id database)}
+ :model/Field _f1 {:table_id (:id table)}
+ :model/Field _f2 {:table_id (:id table)}
+ :model/Field _f3 {:table_id (:id table)}
+ :model/Field _f4 {:table_id (:id table)}
+ :model/Field _f5 {:table_id (:id table)}]
+ (mt/with-temporary-setting-values [interestingness-max-fields-per-sync 2]
+ (let [stats (#'sync.interestingness/score-missing-leftovers! database)]
+ (is (= 2 (:fields-scored stats)))
+ (is (= 0 (:fields-failed stats)))
+ (is (= 3 (:fields-remaining stats))
+ "a truncated run reports how many fields of this database are still unscored")))
+ (testing "the scores written before the limit hit are committed, not rolled back"
+ (is (= 3 (t2/count :model/Field :table_id (:id table) :dimension_interestingness nil))))
+ (testing "the next run continues on the remainder rather than starting over"
+ (mt/with-temporary-setting-values [interestingness-max-fields-per-sync 100]
+ (is (= {:fields-scored 3 :fields-failed 0}
+ (#'sync.interestingness/score-missing-leftovers! database)))))
+ (testing "and a run with nothing left to do reports no remainder"
+ (is (= {:fields-scored 0 :fields-failed 0}
+ (#'sync.interestingness/score-missing-leftovers! database)))))))
+
+(deftest score-missing-leftovers-keeps-going-past-a-failing-field-test
+ (testing "a field whose scoring throws does not stop the rest of the run"
+ (mt/with-temp [:model/Database database {}
+ :model/Table table {:db_id (:id database)}
+ :model/Field f1 {:table_id (:id table) :name "aaa"}
+ :model/Field f2 {:table_id (:id table) :name "boom"}
+ :model/Field f3 {:table_id (:id table) :name "zzz"}]
+ (mt/with-dynamic-fn-redefs [interestingness/dimension-interestingness
+ (fn [field]
+ (if (= "boom" (:name field))
+ (throw (ex-info "boom" {}))
+ 0.5))]
+ (with-group-size! 1
+ (fn []
+ (is (= {:fields-scored 2 :fields-failed 1}
+ (#'sync.interestingness/score-missing-leftovers! database))
+ "the failing field must not halt the pass, nor stall the cursor on itself"))))
+ (is (= [0.5 nil 0.5] (map score-of [f1 f2 f3]))))))
+
+(deftest score-missing-leftovers-persists-the-canonical-score-test
+ (testing "the pass persists exactly what the scorer computes for the field"
+ (let [fingerprint {:global {:distinct-count 5000 :nil% 0.0}
+ :type {:type/DateTime {:earliest "2022-01-01" :latest "2024-12-31"}}}]
+ (mt/with-temp [:model/Database database {}
+ :model/Table table {:db_id (:id database)}
+ :model/Field field {:table_id (:id table)
+ :base_type :type/DateTime
+ :semantic_type :type/CreationTimestamp
+ :fingerprint fingerprint}]
+ (is (= {:fields-scored 1 :fields-failed 0}
+ (#'sync.interestingness/score-missing-leftovers! database)))
+ (is (= (interestingness/dimension-interestingness
+ (t2/select-one :model/Field :id (:id field)))
+ (score-of field)))))))
+
+(deftest score-missing-leftovers-respects-user-set-visibility-test
+ (testing "a Field hidden by a user is not scored, as users' visibility_type is what the pass selects on"
+ (mt/with-temp [:model/Database database {}
+ :model/Table table {:db_id (:id database)}
+ :model/Field visible {:table_id (:id table)}
+ :model/Field hidden {:table_id (:id table)}]
+ (t2/insert! :model/FieldUserSettings {:field_id (:id hidden)
+ :visibility_type :sensitive})
+ (is (= {:fields-scored 1 :fields-failed 0}
+ (#'sync.interestingness/score-missing-leftovers! database)))
+ (is (some? (score-of visible)))
+ (is (nil? (score-of hidden))
+ "the user-set visibility_type must still exclude the field after the rewrite"))))