Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[deps] After a sync, re-analyze all dependents of updated tables (#68621)

Braden Shepherdson3d4e44b77658
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
index 4165000a7ac..0056f61c220 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
@@ -276,20 +276,23 @@
(when (premium-features/has-feature? :dependencies)
(t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))))
-(defn- check-dependents! [type object recur-through-transforms?]
+(defn- check-all-dependents! [type->objects recur-through-transforms?]
(let [graph (if recur-through-transforms?
(models.dependency/graph-dependents)
(models.dependency/filtered-graph-dependents
nil
(fn [type-field _id-field]
[:not= type-field "transform"])))
- children-map (models.dependency/transitive-mbql-dependents graph {type [object]})]
+ children-map (models.dependency/transitive-mbql-dependents graph type->objects)]
(doseq [[type children] children-map
:when (deps.findings/supported-entities type)
instances (partition 50 50 nil children)]
(-> (t2/select (deps.dependency-types/dependency-type->model type) :id [:in instances])
deps.findings/analyze-instances!))))
+(defn- check-dependents! [type object recur-through-transforms?]
+ (check-all-dependents! {type [object]} recur-through-transforms?))
+
(derive ::check-card-dependents :metabase/event)
(derive :event/card-create ::check-card-dependents)
(derive :event/card-update ::check-card-dependents)
@@ -335,3 +338,49 @@
(when (premium-features/has-feature? :dependencies)
(lib-be/with-metadata-provider-cache
(check-dependents! :transform {:id (:transform-id object)} true))))
+
+(defn- synced-db->direct-dependents-of-changed-tables
+ "Given the `:db_id` of a freshly synced database, this examines all tables in the DB which were updated, or have
+ fields which were updated, since the last time any cards depending on them were analyzed.
+
+ It is important that this doesn't re-run the analysis for all dependents of every table whose DB got synced -
+ most of the tables have no change every time.
+
+ Returns the set of table IDs which have dependents that need re-analysis, possibly empty."
+ [db-id]
+ (t2/select-fn-set :table_id :model/AnalysisFinding
+ {:select [:field_updates/table_id]
+ :from [[{:select [[:table/id :table_id]
+ [:table/updated_at :last_table_update]
+ [[:max :field/updated_at] :last_field_update]]
+ :from [[(t2/table-name :model/Table) :table]]
+ :left-join [[(t2/table-name :model/Field) :field]
+ [:= :field/table_id :table/id]]
+ :where [:= :table/db_id db-id]
+ :group-by [:table/id
+ :table/updated_at]}
+ :field_updates]]
+ :inner-join [[(t2/table-name :model/Dependency) :dep]
+ [:and
+ [:= :dep/to_entity_type [:inline "table"]]
+ [:= :field_updates/table_id :dep/to_entity_id]]
+ [(t2/table-name :model/AnalysisFinding) :finding]
+ [:and
+ [:= :finding/analyzed_entity_type :dep/from_entity_type]
+ [:= :finding/analyzed_entity_id :dep/from_entity_id]]]
+ :where [:and
+ [:!= :finding/analyzed_entity_id nil]
+ [:or
+ [:< :finding/analyzed_at :field_updates/last_table_update]
+ [:< :finding/analyzed_at :field_updates/last_field_update]]]}))
+
+(derive ::sync-completed-on-database :metabase/event)
+(derive :event/sync-end ::sync-completed-on-database)
+
+(methodical/defmethod events/publish-event! ::sync-completed-on-database
+ [_ {db-id :database_id}]
+ (when (premium-features/has-feature? :dependencies)
+ (lib-be/with-metadata-provider-cache
+ (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]
+ (when (seq changes)
+ (check-all-dependents! {:table changes} true))))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index 9531b0f82a5..f5931ea1a94 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -157,12 +157,16 @@
(filtered-graph key-dependents destination-filter-fn source-filter-fn)))
(defn entities->nodes
- "Converts a map of entities `{entity-type [{:id 1, ...} ...]}` into a list of nodes `[[entity-type entity-id]]`."
+ "Converts a map of entities `{entity-type [{:id 1, ...} ...]}` or entity IDs `{entity-type [1]}` into a list of nodes
+ `[[entity-type entity-id]]`."
[entities-map]
(for [[entity-type entities] entities-map
entity entities
- :when (:id entity)]
- [entity-type (:id entity)]))
+ :let [id (if (number? entity)
+ entity
+ (:id entity))]
+ :when id]
+ [entity-type id]))
(defn group-nodes
"Groups a list of nodes `[[entity-type entity-id]]` by their type."
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
index 4b36c0cb476..09cd1929ed2 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
@@ -1,13 +1,22 @@
(ns metabase-enterprise.dependencies.metadata-update-test
(:require
+ [clojure.java.jdbc :as jdbc]
[clojure.test :refer [deftest is testing]]
+ [java-time.api :as t]
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
+ [metabase-enterprise.dependencies.events :as deps.events]
+ [metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.metadata-update :as deps.metadata-update]
+ [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]
[metabase.api.common :as api]
[metabase.events.core :as events]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.sync.core :as sync]
[metabase.test :as mt]
+ [metabase.test.data.one-off-dbs :as one-off-dbs]
+ [metabase.util :as u]
[toucan2.core :as t2]))
(defn assert-traverses-grandchild [{:keys [grandchild-dependency-type grandchild-data should-traverse? should-include?]}]
@@ -273,3 +282,138 @@
(t2/select-one-fn #(get-in % [:result_metadata 0 :display_name])
[:model/Card :id :result_metadata :card_schema]
:id grandchild-id))))))))))
+
+(defn- with-syncable-db! [thunk]
+ (mt/with-premium-features #{:dependencies}
+ (one-off-dbs/with-blank-db
+ ;; Step 1: Create a table with two columns
+ (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"
+ "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]
+ (jdbc/execute! one-off-dbs/*conn* [statement]))
+
+ ;; Sync the database to pick up the new table
+ (sync/sync-database! (mt/db))
+
+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")
+ filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")
+ ;; Step 2: Create a card with a filter on filter_col
+ mp (lib-be/application-database-metadata-provider (mt/id))
+ table (lib.metadata/table mp table-id)
+ filter-field (lib.metadata/field mp filter-field-id)
+ query (-> (lib/query mp table)
+ (lib/filter (lib/= filter-field "value1")))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query
+ :database_id (mt/id)}]
+ ;; Create the dependency manually
+ (t2/insert! :model/Dependency {:from_entity_type :card
+ :from_entity_id card-id
+ :to_entity_type :table
+ :to_entity_id table-id})
+
+ ;; Step 3: Run analysis - should succeed with no errors
+ (let [card (t2/select-one :model/Card card-id)]
+ ;; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see
+ ;; old values after the sync!
+ (lib-be/with-metadata-provider-cache
+ (deps.findings/upsert-analysis! card)))
+
+ (testing "Initial analysis succeeds"
+ (is (true?
+ (t2/select-one-fn :result :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)))
+ (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))
+
+ ;; Backdate the analysis timestamp and table/field updated_at so
+ ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)
+ ;; after a new sync makes an edit.
+ ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and
+ ;; the field update from sync would have identical timestamps.
+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]
+ (t2/update! :model/AnalysisFinding
+ {:analyzed_entity_type :card :analyzed_entity_id card-id}
+ {:analyzed_at old-time})
+ (t2/update! :model/Field :table_id table-id
+ {:updated_at old-time})
+ (t2/update! :model/Table :id table-id
+ {:updated_at old-time}))
+
+ ;; Setup complete: Now call the thunk for the actual test run.
+ (thunk {:db-id (:id (mt/db))
+ :card-id card-id
+ :table-id table-id
+ :filter-field-id filter-field-id}))))))
+
+;; Integration test that a real DB sync triggers re-analysis of an updated table.
+(deftest ^:sequential sync-removed-column-triggers-reanalysis-test
+ (testing "When sync detects a removed column, re-analyzing the card shows errors"
+ (with-syncable-db!
+ (fn [{:keys [card-id filter-field-id]}]
+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ ;; Step 4: Remove the column used in the filter
+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
+
+ ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis
+ ;; (because analyzed_at < field.updated_at) and trigger re-analysis.
+ (sync/sync-database! (mt/db))
+
+ ;; Verify the field is now inactive
+ (testing "Field is marked inactive after sync"
+ (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))
+
+ ;; Step 6 & 7: The sync-end event should have triggered re-analysis.
+ ;; Check that analysis now fails with missing column error.
+ (testing "Re-analysis detects the missing column"
+ (let [finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ (is (some? finding) "Analysis finding should exist")
+ (is (false? (:result finding)) "Analysis should fail")
+ (is (not= initial-analysis-time (:analyzed_at finding))
+ "Analysis should have been re-run"))
+
+ (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]
+ (is (seq errors) "Should have analysis errors")
+ (is (some #(= :missing-column (:error_type %)) errors)
+ "Should have a missing-column error"))))))))
+
+;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all
+;; cards which depend on that table.
+(deftest ^:sequential sync-without-changes-does-not-trigger-reanalysis-test
+ (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"
+ (with-syncable-db!
+ (fn [{:keys [card-id db-id filter-field-id table-id]}]
+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)
+ db-deps-checked (atom [])
+ original-deps-check @#'deps.events/synced-db->direct-dependents-of-changed-tables
+ table-before (into {} (t2/select-one :model/Table :id table-id))
+ filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]
+ ;; Re-sync the database, having made no changes to it.
+ (with-redefs [deps.events/synced-db->direct-dependents-of-changed-tables
+ (fn [db-id]
+ (u/prog1 (original-deps-check db-id)
+ (swap! db-deps-checked conj [db-id <>])))]
+ (sync/sync-database! (mt/db)))
+
+ (testing "sync doesn't update tables or fields that haven't changed"
+ (let [table-after (into {} (t2/select-one :model/Table :id table-id))
+ filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]
+ (is (= table-before table-after))
+ (is (= filter-field-before filter-field-after))))
+
+ (testing "the DB is were checked for tables that need deps updates, but none were found"
+ (is (=? [[db-id empty?]]
+ @db-deps-checked)))
+
+ (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"
+ (let [finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ (is (some? finding) "Analysis finding should exist")
+ (is (true? (:result finding)) "Analysis should be a success")
+ (is (= initial-analysis-time (:analyzed_at finding))
+ "Analysis should not have been re-run"))))))))