The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[deps] After a sync, re-analyze all dependents of updated tables (#68621)
Braden Shepherdson3d4e44b77658
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.cljindex 4165000a7ac..0056f61c220 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj@@ -276,20 +276,23 @@(when (premium-features/has-feature? :dependencies)(t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))))-(defn- check-dependents! [type object recur-through-transforms?]+(defn- check-all-dependents! [type->objects recur-through-transforms?](let [graph (if recur-through-transforms?(models.dependency/graph-dependents)(models.dependency/filtered-graph-dependentsnil(fn [type-field _id-field][:not= type-field "transform"])))- children-map (models.dependency/transitive-mbql-dependents graph {type [object]})]+ children-map (models.dependency/transitive-mbql-dependents graph type->objects)](doseq [[type children] children-map:when (deps.findings/supported-entities type)instances (partition 50 50 nil children)](-> (t2/select (deps.dependency-types/dependency-type->model type) :id [:in instances])deps.findings/analyze-instances!))))+(defn- check-dependents! [type object recur-through-transforms?]+ (check-all-dependents! {type [object]} recur-through-transforms?))+(derive ::check-card-dependents :metabase/event)(derive :event/card-create ::check-card-dependents)(derive :event/card-update ::check-card-dependents)@@ -335,3 +338,49 @@(when (premium-features/has-feature? :dependencies)(lib-be/with-metadata-provider-cache(check-dependents! :transform {:id (:transform-id object)} true))))++(defn- synced-db->direct-dependents-of-changed-tables+ "Given the `:db_id` of a freshly synced database, this examines all tables in the DB which were updated, or have+ fields which were updated, since the last time any cards depending on them were analyzed.++ It is important that this doesn't re-run the analysis for all dependents of every table whose DB got synced -+ most of the tables have no change every time.++ Returns the set of table IDs which have dependents that need re-analysis, possibly empty."+ [db-id]+ (t2/select-fn-set :table_id :model/AnalysisFinding+ {:select [:field_updates/table_id]+ :from [[{:select [[:table/id :table_id]+ [:table/updated_at :last_table_update]+ [[:max :field/updated_at] :last_field_update]]+ :from [[(t2/table-name :model/Table) :table]]+ :left-join [[(t2/table-name :model/Field) :field]+ [:= :field/table_id :table/id]]+ :where [:= :table/db_id db-id]+ :group-by [:table/id+ :table/updated_at]}+ :field_updates]]+ :inner-join [[(t2/table-name :model/Dependency) :dep]+ [:and+ [:= :dep/to_entity_type [:inline "table"]]+ [:= :field_updates/table_id :dep/to_entity_id]]+ [(t2/table-name :model/AnalysisFinding) :finding]+ [:and+ [:= :finding/analyzed_entity_type :dep/from_entity_type]+ [:= :finding/analyzed_entity_id :dep/from_entity_id]]]+ :where [:and+ [:!= :finding/analyzed_entity_id nil]+ [:or+ [:< :finding/analyzed_at :field_updates/last_table_update]+ [:< :finding/analyzed_at :field_updates/last_field_update]]]}))++(derive ::sync-completed-on-database :metabase/event)+(derive :event/sync-end ::sync-completed-on-database)++(methodical/defmethod events/publish-event! ::sync-completed-on-database+ [_ {db-id :database_id}]+ (when (premium-features/has-feature? :dependencies)+ (lib-be/with-metadata-provider-cache+ (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]+ (when (seq changes)+ (check-all-dependents! {:table changes} true))))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex 9531b0f82a5..f5931ea1a94 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -157,12 +157,16 @@(filtered-graph key-dependents destination-filter-fn source-filter-fn)))(defn entities->nodes- "Converts a map of entities `{entity-type [{:id 1, ...} ...]}` into a list of nodes `[[entity-type entity-id]]`."+ "Converts a map of entities `{entity-type [{:id 1, ...} ...]}` or entity IDs `{entity-type [1]}` into a list of nodes+ `[[entity-type entity-id]]`."[entities-map](for [[entity-type entities] entities-mapentity entities- :when (:id entity)]- [entity-type (:id entity)]))+ :let [id (if (number? entity)+ entity+ (:id entity))]+ :when id]+ [entity-type id]))(defn group-nodes"Groups a list of nodes `[[entity-type entity-id]]` by their type."diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.cljindex 4b36c0cb476..09cd1929ed2 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj@@ -1,13 +1,22 @@(ns metabase-enterprise.dependencies.metadata-update-test(:require+ [clojure.java.jdbc :as jdbc][clojure.test :refer [deftest is testing]]+ [java-time.api :as t][metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.dependencies.events :as deps.events]+ [metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.metadata-update :as deps.metadata-update]+ [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error][metabase.api.common :as api][metabase.events.core :as events]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.sync.core :as sync][metabase.test :as mt]+ [metabase.test.data.one-off-dbs :as one-off-dbs]+ [metabase.util :as u][toucan2.core :as t2]))(defn assert-traverses-grandchild [{:keys [grandchild-dependency-type grandchild-data should-traverse? should-include?]}]@@ -273,3 +282,138 @@(t2/select-one-fn #(get-in % [:result_metadata 0 :display_name])[:model/Card :id :result_metadata :card_schema]:id grandchild-id))))))))))++(defn- with-syncable-db! [thunk]+ (mt/with-premium-features #{:dependencies}+ (one-off-dbs/with-blank-db+ ;; Step 1: Create a table with two columns+ (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"+ "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]+ (jdbc/execute! one-off-dbs/*conn* [statement]))++ ;; Sync the database to pick up the new table+ (sync/sync-database! (mt/db))++ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")+ filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")+ ;; Step 2: Create a card with a filter on filter_col+ mp (lib-be/application-database-metadata-provider (mt/id))+ table (lib.metadata/table mp table-id)+ filter-field (lib.metadata/field mp filter-field-id)+ query (-> (lib/query mp table)+ (lib/filter (lib/= filter-field "value1")))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query+ :database_id (mt/id)}]+ ;; Create the dependency manually+ (t2/insert! :model/Dependency {:from_entity_type :card+ :from_entity_id card-id+ :to_entity_type :table+ :to_entity_id table-id})++ ;; Step 3: Run analysis - should succeed with no errors+ (let [card (t2/select-one :model/Card card-id)]+ ;; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see+ ;; old values after the sync!+ (lib-be/with-metadata-provider-cache+ (deps.findings/upsert-analysis! card)))++ (testing "Initial analysis succeeds"+ (is (true?+ (t2/select-one-fn :result :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)))+ (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))++ ;; Backdate the analysis timestamp and table/field updated_at so+ ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)+ ;; after a new sync makes an edit.+ ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and+ ;; the field update from sync would have identical timestamps.+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]+ (t2/update! :model/AnalysisFinding+ {:analyzed_entity_type :card :analyzed_entity_id card-id}+ {:analyzed_at old-time})+ (t2/update! :model/Field :table_id table-id+ {:updated_at old-time})+ (t2/update! :model/Table :id table-id+ {:updated_at old-time}))++ ;; Setup complete: Now call the thunk for the actual test run.+ (thunk {:db-id (:id (mt/db))+ :card-id card-id+ :table-id table-id+ :filter-field-id filter-field-id}))))))++;; Integration test that a real DB sync triggers re-analysis of an updated table.+(deftest ^:sequential sync-removed-column-triggers-reanalysis-test+ (testing "When sync detects a removed column, re-analyzing the card shows errors"+ (with-syncable-db!+ (fn [{:keys [card-id filter-field-id]}]+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ ;; Step 4: Remove the column used in the filter+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])++ ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis+ ;; (because analyzed_at < field.updated_at) and trigger re-analysis.+ (sync/sync-database! (mt/db))++ ;; Verify the field is now inactive+ (testing "Field is marked inactive after sync"+ (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))++ ;; Step 6 & 7: The sync-end event should have triggered re-analysis.+ ;; Check that analysis now fails with missing column error.+ (testing "Re-analysis detects the missing column"+ (let [finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ (is (some? finding) "Analysis finding should exist")+ (is (false? (:result finding)) "Analysis should fail")+ (is (not= initial-analysis-time (:analyzed_at finding))+ "Analysis should have been re-run"))++ (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]+ (is (seq errors) "Should have analysis errors")+ (is (some #(= :missing-column (:error_type %)) errors)+ "Should have a missing-column error"))))))))++;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all+;; cards which depend on that table.+(deftest ^:sequential sync-without-changes-does-not-trigger-reanalysis-test+ (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"+ (with-syncable-db!+ (fn [{:keys [card-id db-id filter-field-id table-id]}]+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)+ db-deps-checked (atom [])+ original-deps-check @#'deps.events/synced-db->direct-dependents-of-changed-tables+ table-before (into {} (t2/select-one :model/Table :id table-id))+ filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]+ ;; Re-sync the database, having made no changes to it.+ (with-redefs [deps.events/synced-db->direct-dependents-of-changed-tables+ (fn [db-id]+ (u/prog1 (original-deps-check db-id)+ (swap! db-deps-checked conj [db-id <>])))]+ (sync/sync-database! (mt/db)))++ (testing "sync doesn't update tables or fields that haven't changed"+ (let [table-after (into {} (t2/select-one :model/Table :id table-id))+ filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]+ (is (= table-before table-after))+ (is (= filter-field-before filter-field-after))))++ (testing "the DB is were checked for tables that need deps updates, but none were found"+ (is (=? [[db-id empty?]]+ @db-deps-checked)))++ (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"+ (let [finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ (is (some? finding) "Analysis finding should exist")+ (is (true? (:result finding)) "Analysis should be a success")+ (is (= initial-analysis-time (:analyzed_at finding))+ "Analysis should not have been re-run"))))))))