The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Push the analysis of the dependents to the job (#68749)
metamben65189a0b69ee
diff --git a/e2e/support/helpers/e2e-dependency-helpers.ts b/e2e/support/helpers/e2e-dependency-helpers.tsindex de761f83d47..3a34092b021 100644--- a/e2e/support/helpers/e2e-dependency-helpers.ts+++ b/e2e/support/helpers/e2e-dependency-helpers.ts@@ -1,3 +1,8 @@+import type {+ DependencyNode,+ ListBrokenGraphNodesResponse,+} from "metabase-types/api";+export const DependencyGraph = {graph: () => cy.findByTestId("dependency-graph"),entryButton: () => cy.findByTestId("graph-entry-button"),@@ -29,3 +34,27 @@ export const DependencyDiagnostics = {cy.findByRole("region", { name: "Broken dependents" }),},};++const WAIT_TIMEOUT = 10000;+const WAIT_INTERVAL = 100;++export function waitForBrokenDependencies(+ filter: (nodes: DependencyNode[]) => boolean,+ timeout = WAIT_TIMEOUT,+): Cypress.Chainable {+ return cy+ .request<ListBrokenGraphNodesResponse>(+ "GET",+ "/api/ee/dependencies/graph/broken",+ )+ .then((response) => {+ if (filter(response.body.data)) {+ return cy.wrap(response);+ } else if (timeout > 0) {+ cy.wait(WAIT_INTERVAL);+ return waitForBrokenDependencies(filter, timeout - WAIT_INTERVAL);+ } else {+ throw new Error("Dependency analysis retry timeout");+ }+ });+}diff --git a/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.tsindex 3c255b78a7c..14afa71fb07 100644--- a/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.ts+++ b/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.ts@@ -8,23 +8,23 @@ const TABLE_NAME = "test_transform_table";const TABLE_DISPLAY_NAME = "Test Transform Table";const TABLE_TRANSFORM = "Test Transform";const TABLE_BASED_QUESTION_BROKEN_FIELD =- "Table-based question with broken field";+ "Test Table-based question with broken field";const TABLE_BASED_QUESTION_BROKEN_EXPRESSION =- "Table-based question with broken expression";+ "Test Table-based question with broken expression";const TABLE_BASED_QUESTION_BROKEN_FILTER =- "Table-based question with broken filter";+ "Test Table-based question with broken filter";const TABLE_BASED_QUESTION_BROKEN_BREAKOUT =- "Table-based question with broken breakout";+ "Test Table-based question with broken breakout";const TABLE_BASED_QUESTION_BROKEN_AGGREGATION =- "Table-based question with broken aggregation";+ "Test Table-based question with broken aggregation";const TABLE_BASED_QUESTION_BROKEN_EXPLICIT_JOIN =- "Table-based question with broken explicit join";-const TABLE_BASED_QUESTION = "Table-based question";+ "Test Table-based question with broken explicit join";+const TABLE_BASED_QUESTION = "Test Table-based question";const QUESTION_BASED_QUESTION_BROKEN_FILTER =- "Question-based question with broken filter";-const TABLE_BASED_MODEL = "Table-based model";+ "Test Question-based question with broken filter";+const TABLE_BASED_MODEL = "Test Table-based model";const MODEL_BASED_MODEL_BROKEN_AGGREGATION =- "Model-based model with broken aggregation";+ "Test Model-based model with broken aggregation";const BROKEN_TABLE_DEPENDENCIES = [TABLE_DISPLAY_NAME];const BROKEN_TABLE_DEPENDENTS = [@@ -206,6 +206,7 @@ describe("scenarios > dependencies > broken list", () => {describe("sorting", () => {it("should sort by name", () => {H.DependencyDiagnostics.visitBrokenDependencies();+ H.DependencyDiagnostics.searchInput().type("test");cy.log("sorted by name by default");checkListSorting({@@ -227,6 +228,7 @@ describe("scenarios > dependencies > broken list", () => {it("should sort by location", () => {H.DependencyDiagnostics.visitBrokenDependencies();+ H.DependencyDiagnostics.searchInput().type("test");cy.log("sorted by location ascending");H.DependencyDiagnostics.list().findByText("Location").click();@@ -243,6 +245,7 @@ describe("scenarios > dependencies > broken list", () => {it("should sort by dependents errors", () => {H.DependencyDiagnostics.visitBrokenDependencies();+ H.DependencyDiagnostics.searchInput().type("test");cy.log("sorted by dependents errors ascending");H.DependencyDiagnostics.list().findByText("Problems").click();@@ -261,6 +264,7 @@ describe("scenarios > dependencies > broken list", () => {it("should sort by dependents with errors", () => {H.DependencyDiagnostics.visitBrokenDependencies();+ H.DependencyDiagnostics.searchInput().type("test");cy.log("sorted by dependents with errors ascending");H.DependencyDiagnostics.list().findByText("Broken dependents").click();@@ -285,6 +289,7 @@ function createContent() {createQuestionContent();createModelContent();breakTransform();+ waitForBrokenDependencies();}function dropTransformTable() {@@ -350,6 +355,8 @@ function breakTransform() {function createTableContent() {H.getTableId({ name: TABLE_NAME }).then((tableId) => {+ cy.wrap(tableId).as("tableId");+H.getFieldId({ tableId, name: "score" }).then((scoreFieldId) => {H.getFieldId({ tableId, name: "status" }).then((statusFieldId) => {H.createQuestion({@@ -442,6 +449,7 @@ function createQuestionContent() {},collection_id: ADMIN_PERSONAL_COLLECTION_ID,}).then(({ body: card }) => {+ cy.wrap(card.id).as("questionId");H.createQuestion({name: QUESTION_BASED_QUESTION_BROKEN_FILTER,query: {@@ -463,6 +471,7 @@ function createModelContent() {"source-table": tableId,},}).then(({ body: card }) => {+ cy.wrap(card.id).as("modelId");H.createQuestion({name: MODEL_BASED_MODEL_BROKEN_AGGREGATION,type: "model",@@ -477,6 +486,12 @@ function createModelContent() {});}+function waitForBrokenDependencies() {+ H.waitForBrokenDependencies(+ (nodes) => nodes.length >= BROKEN_DEPENDENCIES.length,+ );+}+function checkList({visibleEntities = [],hiddenEntities = [],diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.cljindex 99561aa48f8..44d0c65476f 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj@@ -1,9 +1,9 @@(ns metabase-enterprise.dependencies.events(:require[metabase-enterprise.dependencies.calculation :as deps.calculation]- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types][metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check][metabase-enterprise.transforms.core :as transforms][metabase.events.core :as events][metabase.lib-be.core :as lib-be]@@ -277,23 +277,6 @@(when (premium-features/has-feature? :dependencies)(t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))))-(defn- check-all-dependents! [type->objects recur-through-transforms?]- (let [graph (if recur-through-transforms?- (models.dependency/graph-dependents)- (models.dependency/filtered-graph-dependents- nil- (fn [type-field _id-field]- [:not= type-field "transform"])))- children-map (models.dependency/transitive-mbql-dependents graph type->objects)]- (doseq [[type children] children-map- :when (deps.findings/supported-entities type)- instances (partition 50 50 nil children)]- (-> (t2/select (deps.dependency-types/dependency-type->model type) :id [:in instances])- deps.findings/analyze-instances!))))--(defn- check-dependents! [type object recur-through-transforms?]- (check-all-dependents! {type [object]} recur-through-transforms?))-(derive ::check-card-dependents :metabase/event)(derive :event/card-create ::check-card-dependents)(derive :event/card-update ::check-card-dependents)@@ -304,8 +287,11 @@(when (and (premium-features/has-feature? :dependencies)(not (models.dependency/is-native-entity? :card object)))(lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! object)- (check-dependents! :card object false))))+ (let [has-stale-dependents? (t2/with-transaction [_conn]+ (deps.findings/upsert-analysis! object)+ (deps.findings/mark-dependents-stale! :card (:id object)))]+ (when has-stale-dependents?+ (task.entity-check/trigger-entity-check-job!))))))(derive ::check-transform :metabase/event)(derive :event/create-transform ::check-transform)@@ -328,8 +314,11 @@[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)(lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! object)- (check-dependents! :segment object false))))+ (let [has-stale-dependents? (t2/with-transaction [_conn]+ (deps.findings/upsert-analysis! object)+ (deps.findings/mark-dependents-stale! :segment (:id object)))]+ (when has-stale-dependents?+ (task.entity-check/trigger-entity-check-job!))))))(derive ::check-transform-dependents :metabase/event)(derive :event/transform-run-complete ::check-transform-dependents)@@ -337,8 +326,8 @@(methodical/defmethod events/publish-event! ::check-transform-dependents[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (lib-be/with-metadata-provider-cache- (check-dependents! :transform {:id (:transform-id object)} true))))+ (when (deps.findings/mark-dependents-stale! :transform (:transform-id object))+ (task.entity-check/trigger-entity-check-job!))))(defn- synced-db->direct-dependents-of-changed-tables"Given the `:db_id` of a freshly synced database, this examines all tables in the DB which were updated, or have@@ -381,7 +370,7 @@(methodical/defmethod events/publish-event! ::sync-completed-on-database[_ {db-id :database_id}](when (premium-features/has-feature? :dependencies)- (lib-be/with-metadata-provider-cache- (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]- (when (seq changes)- (check-all-dependents! {:table changes} true))))))+ (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]+ (when (and (seq changes)+ (deps.findings/mark-all-dependents-stale! {:table changes}))+ (task.entity-check/trigger-entity-check-job!)))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.cljindex 8e6096ce873..534acb80635 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj@@ -63,12 +63,50 @@"Entities supported by the analysis findings code"#{:card :transform :segment})+(def ^:private SupportedEntityType+ "Schema for entity types supported by analysis findings."+ (into [:enum] supported-entities))++(defn- mark-supported-dependents-stale!+ "Given a map of {entity-type [entity-ids]}, mark all supported types as stale.+ Returns true if any supported dependents were found."+ [dependents]+ (let [supported-dependents (into {}+ (filter (fn [[dep-type dep-ids]]+ (and (supported-entities dep-type)+ (seq dep-ids))))+ dependents)]+ (doseq [[dep-type dep-ids] supported-dependents]+ (deps.analysis-finding/mark-stale! dep-type dep-ids))+ (boolean (seq supported-dependents))))++(mu/defn mark-dependents-stale! :- :boolean+ "Mark all transitive dependents of an entity as stale for re-analysis.+ Returns true if any supported dependents were found, false otherwise.++ Uses `transitive-dependents` (not `transitive-mbql-dependents`) because:+ 1. It's faster - doesn't need to fetch entities to check for native-ness+ 2. Native dependents marked stale will just be quickly re-validated as valid"+ [entity-type :- SupportedEntityType+ entity-id :- pos-int?]+ (let [dependents (models.dependency/transitive-dependents {entity-type [{:id entity-id}]})]+ (mark-supported-dependents-stale! dependents)))++(defn mark-all-dependents-stale!+ "Mark all transitive dependents of multiple entities as stale.+ Takes a map of {entity-type entity-ids} where entity-ids is a set of IDs.+ Returns true if any supported dependents were found."+ [type->entity-ids]+ (let [type->objects (update-vals type->entity-ids #(map (fn [id] {:id id}) %))+ dependents (models.dependency/transitive-dependents type->objects)]+ (mark-supported-dependents-stale! dependents)))+(mu/defn analyze-batch! :- nat-int?"Add or update analyses for a batch of entities.Takes in an entity type and batch size, and looks for a batch of entities with missing or out of dateAnalysisFindings and then upsert new analyses for them."- [type :- (into [:enum] supported-entities)+ [type :- SupportedEntityTypebatch-size :- pos-int?](let [instances (deps.analysis-finding/instances-for-analysis type batch-size)](lib-be/with-metadata-provider-cachediff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.cljindex 41cbdca4460..499aeefd6cf 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj@@ -43,7 +43,8 @@(t2/with-transaction [_conn](let [update {:analyzed_at (mi/now):analysis_version *current-analysis-finding-version*- :result result}+ :result result+ :stale false}existing-id (t2/select-one-fn :id [:model/AnalysisFinding :id]:analyzed_entity_type type:analyzed_entity_id instance-id)]@@ -58,20 +59,45 @@instance-id(map error->finding-error-row finding-details)))))+(def ^:private mark-stale-batch-size+ "Maximum number of entity IDs to process in a single query to avoid parameter limits."+ 1000)++(defn mark-stale!+ "Mark entities of `entity-type` with ids in `entity-ids` and existing findings as stale for re-analysis.+ Entities without existing findings are ignored - they'll be picked up by the normal job flow."+ [entity-type entity-ids]+ (doseq [batch (partition-all mark-stale-batch-size entity-ids)]+ (t2/update! :model/AnalysisFinding+ :analyzed_entity_type entity-type+ :analyzed_entity_id [:in batch]+ {:stale true})))++(defn has-stale-entities?+ "Check if there are any stale analysis records."+ []+ (t2/exists? :model/AnalysisFinding :stale true))+(defn instances-for-analysis- "Find a batch of instances with missing or outdated AnalysisFindings"- [type batch-size]- (let [model (deps.dependency-types/dependency-type->model type)+ "Find a batch of instances of type `entity-type` and maximum size `batch-size` with missing, outdated,+ or stale AnalysisFindings.+ Prioritizes stale entities (stale=true) over outdated version entities."+ [entity-type batch-size]+ (let [model (deps.dependency-types/dependency-type->model entity-type)table-name (t2/table-name model)- id-field (keyword (name table-name) "id")- table-wildcard (keyword (name table-name) "*")]+ id-field (keyword (name table-name) "id")+ table-wildcard (keyword (name table-name) "*")](t2/select model{:select [table-wildcard]:from table-name:left-join [:analysis_finding [:and[:= :analysis_finding.analyzed_entity_id id-field]- [:= :analysis_finding.analyzed_entity_type (name type)]]]- :where [:<- [:coalesce :analysis_finding.analysis_version 0]- *current-analysis-finding-version*]+ [:= :analysis_finding.analyzed_entity_type (name entity-type)]]]+ :where [:or+ [:= :analysis_finding.stale true] ; stale analysis+ [:< ; missing or outdated analysis+ [:coalesce :analysis_finding.analysis_version 0]+ *current-analysis-finding-version*]]+ ;; stale entities first+ :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]]:limit batch-size})))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.cljindex 285502d04a4..0369b378945 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.dependencies.task.entity-check(:require[metabase-enterprise.dependencies.findings :as deps.findings]+ [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding][metabase-enterprise.dependencies.settings :as deps.settings][metabase-enterprise.dependencies.task-util :as deps.task-util][metabase.premium-features.core :as premium-features]@@ -9,19 +10,30 @@(set! *warn-on-reflection* true)+(defn- process-one-batch!+ "Process one batch of entities. Returns true if the full batch was used."+ []+ (-> (reduce (fn [batch-size entity-type]+ (if (< batch-size 1)+ (reduced 0)+ (let [processed (deps.findings/analyze-batch! entity-type batch-size)]+ (when (pos? processed)+ (log/info "Updated" processed "entities of type" entity-type))+ (- batch-size processed))))+ (deps.settings/dependency-entity-check-batch-size)+ deps.findings/supported-entities)+ (< 1)))+(defn- check-entities!+ "Process entities, draining all stale entities before returning.+ Returns true if the full batch was used (more work may remain)."[](when (premium-features/has-feature? :dependencies)- (-> (reduce (fn [batch-size entity-type]- (if (< batch-size 1)- (reduced 0)- (let [processed (deps.findings/analyze-batch! entity-type batch-size)]- (when (pos? processed)- (log/info "Updated" processed "entities of type" entity-type))- (- batch-size processed))))- (deps.settings/dependency-entity-check-batch-size)- deps.findings/supported-entities)- (< 1))))+ (loop []+ (let [used-full-batch? (process-one-batch!)]+ (if (and used-full-batch? (deps.analysis-finding/has-stale-entities?))+ (recur)+ used-full-batch?)))))(declare schedule-next-run!)@@ -56,3 +68,9 @@deps.task-util/job-initial-delayschedule-next-run!)(log/info "Not starting dependency entity check job because the batch size is not positive")))++(defn trigger-entity-check-job!+ "Trigger the DependencyEntityCheck job to run immediately.+ Use this when entities have been marked stale and need re-analysis."+ []+ (schedule-next-run! 0))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex 6199a1471df..c1f485570c3 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -18,15 +18,16 @@(commentmetabase-enterprise.dependencies.events/keep-me)-(defn- run-with-dependencies-setup [thunk]+(defn- run-with-dependencies-setup! [thunk](mt/with-test-user :rasta(mt/with-premium-features #{:dependencies}- (lib-be/with-metadata-provider-cache- (let [mp (mt/metadata-provider)]- (thunk mp))))))+ (mt/with-model-cleanup [:model/AnalysisFinding]+ (lib-be/with-metadata-provider-cache+ (let [mp (mt/metadata-provider)]+ (thunk mp)))))))(deftest dashboard-update-sets-correct-dependencies- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [products-id (mt/id :products)orders-id (mt/id :orders)@@ -143,7 +144,7 @@(is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard))))))))))(deftest document-update-sets-correct-dependencies- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [products-id (mt/id :products)products (lib.metadata/table mp products-id)]@@ -196,7 +197,7 @@(deftest sandbox-update-sets-correct-dependencies(mt/with-premium-features #{:sandboxes}- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "sandbox group"}:model/Card {card1-id :id} {}@@ -226,7 +227,7 @@(deftest card-dependency-calculation-error-handling-test(testing "When dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [products-id (mt/id :products)products (lib.metadata/table mp products-id)]@@ -254,7 +255,7 @@(deftest snippet-dependency-calculation-error-handling-test(testing "When snippet dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](mt/with-temp [:model/NativeQuerySnippet {snippet-id :id :as snippet} {:name "test snippet":content "SELECT 1"}]@@ -281,7 +282,7 @@(deftest ^:sequential native-transform-updates-dependencies-test(testing "native transform update events trigger dependency calculations"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [source {:query (lib/native-query mp "select * from orders"):type :query}]@@ -297,7 +298,7 @@(deftest ^:sequential mbql-transform-updates-dependencies-test(testing "mbql transform update events trigger dependency calculations"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders))):type :query}]@@ -313,7 +314,7 @@(deftest ^:sequential python-transform-updates-dependencies-test(testing "python transform update events trigger dependency calculations"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](let [source {:type :python,:source-database 2,@@ -335,7 +336,7 @@(deftest ^:sequential transform-run-updates-dependencies-test(testing "transform run events trigger dependency calculations"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}](mt/with-temp [:model/Transform {transform-id :id} {:target target}@@ -356,7 +357,7 @@(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test(testing "python transform update events handles downstream dependencies"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](let [source {:type :python,:source-database (mt/id),@@ -401,7 +402,7 @@(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test(testing "query transform update events handles downstream dependencies"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [source {:query (lib/native-query mp "select * from orders"):type :query}@@ -439,7 +440,7 @@(deftest transform-dependency-calculation-error-handling-test(testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](mt/with-temp [:model/Transform {transform-id :id :as transform} {}](log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]@@ -465,7 +466,7 @@(deftest dashboard-dependency-calculation-error-handling-test(testing "When dashboard dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](mt/with-temp [:model/Dashboard {dashboard-id :id :as dashboard} {}](log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]@@ -491,7 +492,7 @@(deftest document-dependency-calculation-error-handling-test(testing "When document dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](mt/with-temp [:model/Document {document-id :id :as document} {:content_type "application/json+vnd.prose-mirror":document {:type "doc" :content []}}]@@ -519,7 +520,7 @@(deftest sandbox-dependency-calculation-error-handling-test(testing "When sandbox dependency calculation throws an error, it should be logged and the version should still be updated"(mt/with-premium-features #{:sandboxes}- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "test group"}:model/Card {card-id :id} {}@@ -548,7 +549,7 @@(is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))))))(deftest segment-update-sets-correct-dependencies- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](let [products-id (mt/id :products)price-field-id (mt/id :products :price)@@ -579,7 +580,7 @@(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))))(deftest card-with-segment-dependencies- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](let [products-id (mt/id :products)price-field-id (mt/id :products :price)]@@ -605,7 +606,7 @@(deftest segment-dependency-calculation-error-handling-test(testing "When segment dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [_](let [products-id (mt/id :products)price-field-id (mt/id :products :price)]@@ -633,7 +634,7 @@(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))))))))(deftest measure-update-sets-correct-dependencies- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](mt/with-test-user :rasta(let [orders-id (mt/id :orders)@@ -668,7 +669,7 @@(is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))(deftest measure-with-measure-dependencies- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [orders-id (mt/id :orders)orders (lib.metadata/table mp orders-id)@@ -697,7 +698,7 @@(t2/select :model/Dependency :from_entity_id measure2-id :from_entity_type :measure)))))))))))))(deftest card-with-measure-dependencies- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [orders-id (mt/id :orders)orders (lib.metadata/table mp orders-id)@@ -725,7 +726,7 @@(deftest measure-dependency-calculation-error-handling-test(testing "When measure dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](let [orders-id (mt/id :orders)orders (lib.metadata/table mp orders-id)@@ -774,9 +775,9 @@(is (= expected-version (analyses-map id)))))))))(deftest ^:sequential card-update-works-with-no-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp]- (testing "Card updates should publish new analyses when no existing ones exist"+ (testing "Card updates should analyze the card and mark dependents stale for async processing"(let [products-id (mt/id :products)products (lib.metadata/table mp products-id)](mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}@@ -787,13 +788,16 @@:to_entity_id parent-card-id}](events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})(assert-has-analyses- {:card {parent-card-id -1- child-card-id -1}})))))))+ {:card {parent-card-id -1}})+ (is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id child-card-id))+ "entities without analysis findings should not be marked: no fake analysis record should be created")))))))(deftest ^:sequential card-update-updates-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp]- (testing "Card updates should update existing analyses when they exist"+ (testing "Card updates should update the card's analysis and mark dependents stale"(let [products-id (mt/id :products)orders-id (mt/id :orders)products (lib.metadata/table mp products-id)@@ -815,16 +819,86 @@{:card {parent-card-id old-versionchild-card-id old-versionother-card-id old-version}}))+ (t2/with-transaction [_conn]+ (binding [models.analysis-finding/*current-analysis-finding-version* new-version]+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))+ (testing "Parent should be re-analyzed synchronously"+ (assert-has-analyses+ {:card {parent-card-id new-version}}))+ (testing "Child should be marked stale (not re-analyzed yet)"+ (let [child-finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id child-card-id)]+ (is (= old-version (:analysis_version child-finding)))+ (is (true? (:stale child-finding)))))+ (testing "Other card should be unchanged"+ (let [other-finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id other-card-id)]+ (is (= old-version (:analysis_version other-finding)))+ (is (false? (:stale other-finding))))))))))))++(deftest ^:sequential stale-entities-are-processed-by-job-test+ (run-with-dependencies-setup!+ (fn [mp]+ (testing "Stale entities are re-analyzed when the job runs"+ (let [products (lib.metadata/table mp (mt/id :products))+ current-version models.analysis-finding/*current-analysis-finding-version*]+ (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}+ :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}+ :model/Dependency _ {:from_entity_type :card+ :from_entity_id child-card-id+ :to_entity_type :card+ :to_entity_id parent-card-id}]+ ;; Create initial analyses for both cards+ (deps.findings/upsert-analysis! parent-card)+ (deps.findings/upsert-analysis! child-card)+ ;; Update parent - this marks child as stale+ (t2/with-transaction [_conn]+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ ;; Verify child is stale+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id child-card-id))+ "Child should be marked stale after parent update"))+ ;; Run the job to process stale entities+ (deps.findings/analyze-batch! :card 10)+ ;; Verify child is no longer stale and has been re-analyzed+ (let [child-finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id child-card-id)]+ (is (false? (:stale child-finding))+ "Child should no longer be stale after job runs")+ (is (= current-version (:analysis_version child-finding))+ "Child should have current analysis version"))))))))++(deftest ^:sequential card-update-transaction-rollback-test+ (run-with-dependencies-setup!+ (fn [mp]+ (testing "If marking dependents stale fails, analysis upsert is rolled back"+ (let [products (lib.metadata/table mp (mt/id :products))+ old-version models.analysis-finding/*current-analysis-finding-version*+ new-version (inc old-version)]+ (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]+ ;; Create initial analysis+ (deps.findings/upsert-analysis! card)+ (testing "Initial analysis exists"+ (is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id))))+ ;; Try to update with mark-dependents-stale! throwing an exception(binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))- (testing "Checking that the analyses were updated appropriately"- (assert-has-analyses- {:card {parent-card-id new-version- child-card-id new-version- other-card-id old-version}}))))))))+ (with-redefs [deps.findings/mark-dependents-stale! (fn [_ _] (throw (ex-info "Simulated failure" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated failure"+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})))))+ ;; Verify analysis was NOT updated (rolled back)+ (testing "Analysis should be unchanged after failed event"+ (is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id))))))))))(deftest ^:sequential card-update-ignores-native-cards-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](testing "Native card updates should not trigger analysis"(mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/native-query mp "select * from products")}@@ -839,7 +913,7 @@child-card-id nil}}))))))(deftest ^:sequential card-update-stops-on-transforms-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](testing "Card updates should continue analyzing through a transform"(let [products-id (mt/id :products)@@ -873,15 +947,26 @@child-card-id old-version}:transform {transform-id old-version}}))(binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))- (testing "Checking that analyses were updated correctly"- (assert-has-analyses- {:card {parent-card-id new-version- child-card-id old-version}- :transform {transform-id new-version}}))))))))+ (t2/with-transaction [_conn]+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ (testing "parent card should be re-analyzed synchronously"+ (is (=? {:analysis_version new-version :stale false}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id parent-card-id))))+ (testing "transform (direct dependent) should be marked stale"+ (is (=? {:analysis_version old-version :stale true}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :transform+ :analyzed_entity_id transform-id))))+ (testing "child card (transitive dependent) should be marked stale"+ (is (=? {:analysis_version old-version :stale true}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id child-card-id))))))))))))(deftest ^:sequential transform-update-works-with-no-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](testing "transform update creates new analyses"(let [products-id (mt/id :products)@@ -904,7 +989,7 @@:transform {transform-id -1}})))))))(deftest ^:sequential transform-update-updates-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](testing "transform update updates analyses when they exist"(let [products-id (mt/id :products)@@ -944,7 +1029,7 @@:transform {transform-id new-version}})))))))))(deftest ^:sequential transform-update-ignores-native-transforms-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](testing "Native transform updates should not trigger analysis"(mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query@@ -958,9 +1043,9 @@{:transform {transform-id nil}}))))))(deftest ^:sequential transform-run-works-with-no-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp]- (testing "transform runs create analyses for dependent cards"+ (testing "transform runs work when dependent cards have no pre-existing analyses"(let [products-id (mt/id :products)products (lib.metadata/table mp products-id)](mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}@@ -974,25 +1059,23 @@:from_entity_id card-id:to_entity_type :transform:to_entity_id transform-id}]- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "public"- :output-table "sample"- :transform-id transform-id}- :user-id api/*current-user-id*})- (assert-has-analyses- {:card {card-id -1}})))))))+ ;; Event should complete without error even when there are no pre-existing analyses+ (is (some? (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "public"+ :output-table "sample"+ :transform-id transform-id}+ :user-id api/*current-user-id*})))))))))(deftest ^:sequential transform-run-updates-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](testing "transform runs update existing analyses when they exist"(let [products-id (mt/id :products)orders-id (mt/id :orders)products (lib.metadata/table mp products-id)orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]+ old-version models.analysis-finding/*current-analysis-finding-version*](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}:model/Transform {transform-id :id :as transform} {:source {:type :query:query (lib/query mp products)}@@ -1013,49 +1096,33 @@{:card {card-id old-versionother-card-id old-version}:transform {transform-id old-version}}))- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]+ (t2/with-transaction [_conn](events/publish-event! :event/transform-run-complete{:object {:db-id (mt/id):output-schema "public":output-table "sample":transform-id transform-id}- :user-id api/*current-user-id*}))- (testing "checking that analyses were updated appropriately"- (assert-has-analyses- {:card {card-id new-version- other-card-id old-version}- :transform {transform-id old-version}}))))))))--(deftest ^:sequential transform-run-triggers-analysis-for-native-transforms-test- (run-with-dependencies-setup- (fn [mp]- (testing "Native transform runs should still trigger analysis"- (let [products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id} {:source {:type :query- :query (lib/native-query mp "select * from products")}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :transform- :to_entity_id transform-id}]- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "public"- :output-table "sample"- :transform-id transform-id}- :user-id api/*current-user-id*})- (assert-has-analyses- {:card {card-id -1}})))))))+ :user-id api/*current-user-id*})+ (testing "dependent card should be marked stale"+ (is (=? {:analysis_version old-version :stale true}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id))))+ (testing "other card should be unchanged"+ (is (=? {:analysis_version old-version :stale false}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id other-card-id))))+ (testing "transform should be unchanged"+ (is (=? {:analysis_version old-version :stale false}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :transform+ :analyzed_entity_id transform-id)))))))))))(deftest ^:sequential segment-update-works-with-no-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp]- (testing "segment update creates analyses when necessary"+ (testing "segment update creates analysis for the segment itself"(let [products-id (mt/id :products)price-field-id (mt/id :products :price)products (lib.metadata/table mp products-id)]@@ -1067,12 +1134,12 @@:to_entity_type :segment:to_entity_id segment-id}](events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})+ ;; Segment should be analyzed synchronously(assert-has-analyses- {:card {card-id -1}- :segment {segment-id -1}})))))))+ {:segment {segment-id -1}})))))))(deftest ^:sequential segment-update-updates-analyses-test- (run-with-dependencies-setup+ (run-with-dependencies-setup!(fn [mp](testing "segment update updates analyses when they exist"(let [products-id (mt/id :products)@@ -1093,9 +1160,16 @@(assert-has-analyses{:card {card-id old-version}:segment {segment-id old-version}}))- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*}))- (testing "checking that analyses were updated correctly"- (assert-has-analyses- {:card {card-id new-version}- :segment {segment-id new-version}}))))))))+ (binding [models.analysis-finding/*current-analysis-finding-version* new-version]+ (t2/with-transaction [_conn]+ (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})+ (testing "segment should be re-analyzed synchronously"+ (is (=? {:analysis_version new-version :stale false}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :segment+ :analyzed_entity_id segment-id))))+ (testing "dependent card should be marked stale"+ (is (=? {:analysis_version old-version :stale true}+ (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id))))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.cljindex 314c3971bb6..54ca5c7190c 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj@@ -3,11 +3,15 @@[clojure.test :refer :all][metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][toucan2.core :as t2]))+(use-fixtures :once (fixtures/initialize :db))+(defn- backfill-all-entity-analyses! [](doseq [model [:card :transform :segment]](while (pos? (deps.findings/analyze-batch! model 100)))))@@ -44,11 +48,16 @@(deftest ^:sequential re-analyze-entities-when-analysis-version-bumped-test(backfill-all-entity-analyses!)- (mt/with-premium-features #{:dependencies}- (is (= 0 (deps.findings/analyze-batch! :card 2)))- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]- ;; can't check the actual cards analyzed here because this could be any starting card in the data- (is (= 2 (deps.findings/analyze-batch! :card 2))))))+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))]+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card _ {:dataset_query (lib/query mp products)}+ :model/Card _ {:dataset_query (lib/query mp orders)}]+ (is (= 2 (deps.findings/analyze-batch! :card 2)))+ (is (= 0 (deps.findings/analyze-batch! :card 2)))+ (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]+ (is (= 2 (deps.findings/analyze-batch! :card 2))))))))(deftest ^:sequential does-not-analyze-native-entities-test(testing "assumes native queries are always fine"@@ -62,3 +71,80 @@:model/AnalysisFinding:analyzed_entity_id card-id:analyzed_entity_type :card))))))))++(defn- stale-map+ "Returns a map of {entity-id stale?} for the given card IDs."+ [card-ids]+ (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id [:in card-ids]))++(deftest ^:sequential analyze-batch-picks-up-missing-analyses-test+ (testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFinding]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]+ (testing "card has no analysis finding initially"+ (is (not (t2/exists? :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id))))+ (testing "analyze-batch! creates analysis for the card"+ (lib-be/with-metadata-provider-cache+ (is (pos? (deps.findings/analyze-batch! :card 10)))))+ (testing "card now has an analysis finding"+ (is (t2/exists? :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)))))))))++(deftest mark-dependents-stale-test+ (testing "mark-dependents-stale! marks direct dependents as stale"+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-premium-features #{:dependencies}+ (lib-be/with-metadata-provider-cache+ (mt/with-model-cleanup [:model/AnalysisFinding]+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}+ :model/Dependency _ {:from_entity_type :card+ :from_entity_id child-card-id+ :to_entity_type :card+ :to_entity_id parent-card-id}]+ (run! deps.findings/upsert-analysis!+ (t2/select :model/Card :id [:in [parent-card-id child-card-id]]))+ (is (= {parent-card-id false, child-card-id false}+ (stale-map [parent-card-id child-card-id]))+ "neither should be stale before marking")+ (t2/with-transaction [_conn]+ (deps.findings/mark-dependents-stale! :card parent-card-id)+ (is (= {parent-card-id false, child-card-id true}+ (stale-map [parent-card-id child-card-id]))+ "parent should NOT be stale, child should be stale")))))))))++(deftest mark-dependents-stale-transitive-test+ (testing "mark-dependents-stale! marks transitive dependents as stale"+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-premium-features #{:dependencies}+ (lib-be/with-metadata-provider-cache+ (mt/with-model-cleanup [:model/AnalysisFinding]+ (mt/with-temp [:model/Card {grandparent-id :id} {:dataset_query (lib/query mp products)}+ :model/Card {parent-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp grandparent-id))}+ :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}+ :model/Dependency _ {:from_entity_type :card+ :from_entity_id parent-id+ :to_entity_type :card+ :to_entity_id grandparent-id}+ :model/Dependency _ {:from_entity_type :card+ :from_entity_id child-id+ :to_entity_type :card+ :to_entity_id parent-id}]+ (run! deps.findings/upsert-analysis!+ (t2/select :model/Card :id [:in [grandparent-id parent-id child-id]]))+ (t2/with-transaction [_conn]+ (deps.findings/mark-dependents-stale! :card grandparent-id)+ (is (= {grandparent-id false, parent-id true, child-id true}+ (stale-map [grandparent-id parent-id child-id]))+ "grandparent should NOT be stale, parent and child should be stale (transitive)")))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.cljindex 09cd1929ed2..f7010d0e585 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj@@ -356,28 +356,33 @@(jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"]);; Step 5: Re-sync the database. The sync-end event will detect the stale analysis- ;; (because analyzed_at < field.updated_at) and trigger re-analysis.+ ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.(sync/sync-database! (mt/db));; Verify the field is now inactive(testing "Field is marked inactive after sync"(is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))- ;; Step 6 & 7: The sync-end event should have triggered re-analysis.- ;; Check that analysis now fails with missing column error.- (testing "Re-analysis detects the missing column"- (let [finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- (is (some? finding) "Analysis finding should exist")- (is (false? (:result finding)) "Analysis should fail")- (is (not= initial-analysis-time (:analyzed_at finding))- "Analysis should have been re-run"))-- (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]- (is (seq errors) "Should have analysis errors")- (is (some #(= :missing-column (:error_type %)) errors)- "Should have a missing-column error"))))))))+ ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.+ ;; Due to race conditions, the entity may be either stale (job hasn't run yet)+ ;; or already reanalyzed (job ran). Check for either valid outcome.+ (let [finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ (is (some? finding) "Analysis finding should exist")+ (if (:stale finding)+ ;; Job hasn't run yet - sync marked it stale, that's the expected behavior+ (testing "Entity is marked stale for re-analysis"+ (is (true? (:stale finding))))+ ;; Job already ran - verify the reanalysis detected the missing column+ (testing "Re-analysis detects the missing column"+ (is (false? (:result finding)) "Analysis should fail")+ (is (not= initial-analysis-time (:analyzed_at finding))+ "Analysis should have been re-run")+ (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]+ (is (seq errors) "Should have analysis errors")+ (is (some #(= :missing-column (:error_type %)) errors)+ "Should have a missing-column error"))))))))));; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all;; cards which depend on that table.diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.cljnew file mode 100644index 00000000000..4f5e0759860--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj@@ -0,0 +1,44 @@+(ns metabase-enterprise.dependencies.models.analysis-finding-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(deftest upsert-analysis-clears-stale-flag-test+ (testing "upsert-analysis! sets stale to false"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {}]+ (t2/with-transaction [_conn]+ ;; Insert a finding with stale = true+ (t2/insert! :model/AnalysisFinding+ {:analyzed_entity_type :card+ :analyzed_entity_id card-id+ :analysis_version (dec models.analysis-finding/*current-analysis-finding-version*)+ :analyzed_at (java.time.OffsetDateTime/now)+ :result true+ :stale true})+ ;; Verify it's stale+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)))+ ;; Now upsert a new analysis+ (models.analysis-finding/upsert-analysis! :card card-id true [])+ ;; Verify stale is now false+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id))))))))++(deftest upsert-analysis-new-finding-has-stale-false-test+ (testing "upsert-analysis! creates new findings with stale = false"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {}]+ (t2/with-transaction [_conn]+ ;; Upsert a new analysis (no existing finding)+ (models.analysis-finding/upsert-analysis! :card card-id true [])+ ;; Verify stale is false+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id))))))))diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex 2858201e529..6471e7af8a0 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -804,6 +804,42 @@ databaseChangeLog:tableName: python_libraryconstraintName: unique_python_library_entity_id+ - changeSet:+ id: v59.2026-01-28T10:00:00+ author: metamben+ comment: Add stale column to analysis_finding for asynchronous dependent updates+ changes:+ - addColumn:+ tableName: analysis_finding+ columns:+ - column:+ name: stale+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the entity needs re-analysis due to upstream changes+ constraints:+ nullable: false+ rollback:+ - dropColumn:+ tableName: analysis_finding+ columnName: stale++ - changeSet:+ id: v59.2026-01-28T10:00:01+ author: metamben+ comment: Add partial index on analysis_finding.stale for efficient polling+ changes:+ - sql:+ dbms: postgresql+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale) WHERE stale = true+ - sql:+ dbms: mysql,mariadb,h2+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale)+ rollback:+ - dropIndex:+ tableName: analysis_finding+ indexName: idx_analysis_finding_stale+# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<########################################################################################################################