Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Push the analysis of the dependents to the job (#68749)

metamben65189a0b69ee
diff --git a/e2e/support/helpers/e2e-dependency-helpers.ts b/e2e/support/helpers/e2e-dependency-helpers.ts
index de761f83d47..3a34092b021 100644
--- a/e2e/support/helpers/e2e-dependency-helpers.ts
+++ b/e2e/support/helpers/e2e-dependency-helpers.ts
@@ -1,3 +1,8 @@
+import type {
+ DependencyNode,
+ ListBrokenGraphNodesResponse,
+} from "metabase-types/api";
+
export const DependencyGraph = {
graph: () => cy.findByTestId("dependency-graph"),
entryButton: () => cy.findByTestId("graph-entry-button"),
@@ -29,3 +34,27 @@ export const DependencyDiagnostics = {
cy.findByRole("region", { name: "Broken dependents" }),
},
};
+
+const WAIT_TIMEOUT = 10000;
+const WAIT_INTERVAL = 100;
+
+export function waitForBrokenDependencies(
+ filter: (nodes: DependencyNode[]) => boolean,
+ timeout = WAIT_TIMEOUT,
+): Cypress.Chainable {
+ return cy
+ .request<ListBrokenGraphNodesResponse>(
+ "GET",
+ "/api/ee/dependencies/graph/broken",
+ )
+ .then((response) => {
+ if (filter(response.body.data)) {
+ return cy.wrap(response);
+ } else if (timeout > 0) {
+ cy.wait(WAIT_INTERVAL);
+ return waitForBrokenDependencies(filter, timeout - WAIT_INTERVAL);
+ } else {
+ throw new Error("Dependency analysis retry timeout");
+ }
+ });
+}
diff --git a/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.ts
index 3c255b78a7c..14afa71fb07 100644
--- a/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.ts
+++ b/e2e/test/scenarios/dependencies/dependency-broken-list.cy.spec.ts
@@ -8,23 +8,23 @@ const TABLE_NAME = "test_transform_table";
const TABLE_DISPLAY_NAME = "Test Transform Table";
const TABLE_TRANSFORM = "Test Transform";
const TABLE_BASED_QUESTION_BROKEN_FIELD =
- "Table-based question with broken field";
+ "Test Table-based question with broken field";
const TABLE_BASED_QUESTION_BROKEN_EXPRESSION =
- "Table-based question with broken expression";
+ "Test Table-based question with broken expression";
const TABLE_BASED_QUESTION_BROKEN_FILTER =
- "Table-based question with broken filter";
+ "Test Table-based question with broken filter";
const TABLE_BASED_QUESTION_BROKEN_BREAKOUT =
- "Table-based question with broken breakout";
+ "Test Table-based question with broken breakout";
const TABLE_BASED_QUESTION_BROKEN_AGGREGATION =
- "Table-based question with broken aggregation";
+ "Test Table-based question with broken aggregation";
const TABLE_BASED_QUESTION_BROKEN_EXPLICIT_JOIN =
- "Table-based question with broken explicit join";
-const TABLE_BASED_QUESTION = "Table-based question";
+ "Test Table-based question with broken explicit join";
+const TABLE_BASED_QUESTION = "Test Table-based question";
const QUESTION_BASED_QUESTION_BROKEN_FILTER =
- "Question-based question with broken filter";
-const TABLE_BASED_MODEL = "Table-based model";
+ "Test Question-based question with broken filter";
+const TABLE_BASED_MODEL = "Test Table-based model";
const MODEL_BASED_MODEL_BROKEN_AGGREGATION =
- "Model-based model with broken aggregation";
+ "Test Model-based model with broken aggregation";
const BROKEN_TABLE_DEPENDENCIES = [TABLE_DISPLAY_NAME];
const BROKEN_TABLE_DEPENDENTS = [
@@ -206,6 +206,7 @@ describe("scenarios > dependencies > broken list", () => {
describe("sorting", () => {
it("should sort by name", () => {
H.DependencyDiagnostics.visitBrokenDependencies();
+ H.DependencyDiagnostics.searchInput().type("test");
cy.log("sorted by name by default");
checkListSorting({
@@ -227,6 +228,7 @@ describe("scenarios > dependencies > broken list", () => {
it("should sort by location", () => {
H.DependencyDiagnostics.visitBrokenDependencies();
+ H.DependencyDiagnostics.searchInput().type("test");
cy.log("sorted by location ascending");
H.DependencyDiagnostics.list().findByText("Location").click();
@@ -243,6 +245,7 @@ describe("scenarios > dependencies > broken list", () => {
it("should sort by dependents errors", () => {
H.DependencyDiagnostics.visitBrokenDependencies();
+ H.DependencyDiagnostics.searchInput().type("test");
cy.log("sorted by dependents errors ascending");
H.DependencyDiagnostics.list().findByText("Problems").click();
@@ -261,6 +264,7 @@ describe("scenarios > dependencies > broken list", () => {
it("should sort by dependents with errors", () => {
H.DependencyDiagnostics.visitBrokenDependencies();
+ H.DependencyDiagnostics.searchInput().type("test");
cy.log("sorted by dependents with errors ascending");
H.DependencyDiagnostics.list().findByText("Broken dependents").click();
@@ -285,6 +289,7 @@ function createContent() {
createQuestionContent();
createModelContent();
breakTransform();
+ waitForBrokenDependencies();
}
function dropTransformTable() {
@@ -350,6 +355,8 @@ function breakTransform() {
function createTableContent() {
H.getTableId({ name: TABLE_NAME }).then((tableId) => {
+ cy.wrap(tableId).as("tableId");
+
H.getFieldId({ tableId, name: "score" }).then((scoreFieldId) => {
H.getFieldId({ tableId, name: "status" }).then((statusFieldId) => {
H.createQuestion({
@@ -442,6 +449,7 @@ function createQuestionContent() {
},
collection_id: ADMIN_PERSONAL_COLLECTION_ID,
}).then(({ body: card }) => {
+ cy.wrap(card.id).as("questionId");
H.createQuestion({
name: QUESTION_BASED_QUESTION_BROKEN_FILTER,
query: {
@@ -463,6 +471,7 @@ function createModelContent() {
"source-table": tableId,
},
}).then(({ body: card }) => {
+ cy.wrap(card.id).as("modelId");
H.createQuestion({
name: MODEL_BASED_MODEL_BROKEN_AGGREGATION,
type: "model",
@@ -477,6 +486,12 @@ function createModelContent() {
});
}
+function waitForBrokenDependencies() {
+ H.waitForBrokenDependencies(
+ (nodes) => nodes.length >= BROKEN_DEPENDENCIES.length,
+ );
+}
+
function checkList({
visibleEntities = [],
hiddenEntities = [],
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
index 99561aa48f8..44d0c65476f 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
@@ -1,9 +1,9 @@
(ns metabase-enterprise.dependencies.events
(:require
[metabase-enterprise.dependencies.calculation :as deps.calculation]
- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
[metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
[metabase-enterprise.transforms.core :as transforms]
[metabase.events.core :as events]
[metabase.lib-be.core :as lib-be]
@@ -277,23 +277,6 @@
(when (premium-features/has-feature? :dependencies)
(t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))))
-(defn- check-all-dependents! [type->objects recur-through-transforms?]
- (let [graph (if recur-through-transforms?
- (models.dependency/graph-dependents)
- (models.dependency/filtered-graph-dependents
- nil
- (fn [type-field _id-field]
- [:not= type-field "transform"])))
- children-map (models.dependency/transitive-mbql-dependents graph type->objects)]
- (doseq [[type children] children-map
- :when (deps.findings/supported-entities type)
- instances (partition 50 50 nil children)]
- (-> (t2/select (deps.dependency-types/dependency-type->model type) :id [:in instances])
- deps.findings/analyze-instances!))))
-
-(defn- check-dependents! [type object recur-through-transforms?]
- (check-all-dependents! {type [object]} recur-through-transforms?))
-
(derive ::check-card-dependents :metabase/event)
(derive :event/card-create ::check-card-dependents)
(derive :event/card-update ::check-card-dependents)
@@ -304,8 +287,11 @@
(when (and (premium-features/has-feature? :dependencies)
(not (models.dependency/is-native-entity? :card object)))
(lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! object)
- (check-dependents! :card object false))))
+ (let [has-stale-dependents? (t2/with-transaction [_conn]
+ (deps.findings/upsert-analysis! object)
+ (deps.findings/mark-dependents-stale! :card (:id object)))]
+ (when has-stale-dependents?
+ (task.entity-check/trigger-entity-check-job!))))))
(derive ::check-transform :metabase/event)
(derive :event/create-transform ::check-transform)
@@ -328,8 +314,11 @@
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
(lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! object)
- (check-dependents! :segment object false))))
+ (let [has-stale-dependents? (t2/with-transaction [_conn]
+ (deps.findings/upsert-analysis! object)
+ (deps.findings/mark-dependents-stale! :segment (:id object)))]
+ (when has-stale-dependents?
+ (task.entity-check/trigger-entity-check-job!))))))
(derive ::check-transform-dependents :metabase/event)
(derive :event/transform-run-complete ::check-transform-dependents)
@@ -337,8 +326,8 @@
(methodical/defmethod events/publish-event! ::check-transform-dependents
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (lib-be/with-metadata-provider-cache
- (check-dependents! :transform {:id (:transform-id object)} true))))
+ (when (deps.findings/mark-dependents-stale! :transform (:transform-id object))
+ (task.entity-check/trigger-entity-check-job!))))
(defn- synced-db->direct-dependents-of-changed-tables
"Given the `:db_id` of a freshly synced database, this examines all tables in the DB which were updated, or have
@@ -381,7 +370,7 @@
(methodical/defmethod events/publish-event! ::sync-completed-on-database
[_ {db-id :database_id}]
(when (premium-features/has-feature? :dependencies)
- (lib-be/with-metadata-provider-cache
- (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]
- (when (seq changes)
- (check-all-dependents! {:table changes} true))))))
+ (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]
+ (when (and (seq changes)
+ (deps.findings/mark-all-dependents-stale! {:table changes}))
+ (task.entity-check/trigger-entity-check-job!)))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
index 8e6096ce873..534acb80635 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
@@ -63,12 +63,50 @@
"Entities supported by the analysis findings code"
#{:card :transform :segment})
+(def ^:private SupportedEntityType
+ "Schema for entity types supported by analysis findings."
+ (into [:enum] supported-entities))
+
+(defn- mark-supported-dependents-stale!
+ "Given a map of {entity-type [entity-ids]}, mark all supported types as stale.
+ Returns true if any supported dependents were found."
+ [dependents]
+ (let [supported-dependents (into {}
+ (filter (fn [[dep-type dep-ids]]
+ (and (supported-entities dep-type)
+ (seq dep-ids))))
+ dependents)]
+ (doseq [[dep-type dep-ids] supported-dependents]
+ (deps.analysis-finding/mark-stale! dep-type dep-ids))
+ (boolean (seq supported-dependents))))
+
+(mu/defn mark-dependents-stale! :- :boolean
+ "Mark all transitive dependents of an entity as stale for re-analysis.
+ Returns true if any supported dependents were found, false otherwise.
+
+ Uses `transitive-dependents` (not `transitive-mbql-dependents`) because:
+ 1. It's faster - doesn't need to fetch entities to check for native-ness
+ 2. Native dependents marked stale will just be quickly re-validated as valid"
+ [entity-type :- SupportedEntityType
+ entity-id :- pos-int?]
+ (let [dependents (models.dependency/transitive-dependents {entity-type [{:id entity-id}]})]
+ (mark-supported-dependents-stale! dependents)))
+
+(defn mark-all-dependents-stale!
+ "Mark all transitive dependents of multiple entities as stale.
+ Takes a map of {entity-type entity-ids} where entity-ids is a set of IDs.
+ Returns true if any supported dependents were found."
+ [type->entity-ids]
+ (let [type->objects (update-vals type->entity-ids #(map (fn [id] {:id id}) %))
+ dependents (models.dependency/transitive-dependents type->objects)]
+ (mark-supported-dependents-stale! dependents)))
+
(mu/defn analyze-batch! :- nat-int?
"Add or update analyses for a batch of entities.
Takes in an entity type and batch size, and looks for a batch of entities with missing or out of date
AnalysisFindings and then upsert new analyses for them."
- [type :- (into [:enum] supported-entities)
+ [type :- SupportedEntityType
batch-size :- pos-int?]
(let [instances (deps.analysis-finding/instances-for-analysis type batch-size)]
(lib-be/with-metadata-provider-cache
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
index 41cbdca4460..499aeefd6cf 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
@@ -43,7 +43,8 @@
(t2/with-transaction [_conn]
(let [update {:analyzed_at (mi/now)
:analysis_version *current-analysis-finding-version*
- :result result}
+ :result result
+ :stale false}
existing-id (t2/select-one-fn :id [:model/AnalysisFinding :id]
:analyzed_entity_type type
:analyzed_entity_id instance-id)]
@@ -58,20 +59,45 @@
instance-id
(map error->finding-error-row finding-details)))))
+(def ^:private mark-stale-batch-size
+ "Maximum number of entity IDs to process in a single query to avoid parameter limits."
+ 1000)
+
+(defn mark-stale!
+ "Mark entities of `entity-type` with ids in `entity-ids` and existing findings as stale for re-analysis.
+ Entities without existing findings are ignored - they'll be picked up by the normal job flow."
+ [entity-type entity-ids]
+ (doseq [batch (partition-all mark-stale-batch-size entity-ids)]
+ (t2/update! :model/AnalysisFinding
+ :analyzed_entity_type entity-type
+ :analyzed_entity_id [:in batch]
+ {:stale true})))
+
+(defn has-stale-entities?
+ "Check if there are any stale analysis records."
+ []
+ (t2/exists? :model/AnalysisFinding :stale true))
+
(defn instances-for-analysis
- "Find a batch of instances with missing or outdated AnalysisFindings"
- [type batch-size]
- (let [model (deps.dependency-types/dependency-type->model type)
+ "Find a batch of instances of type `entity-type` and maximum size `batch-size` with missing, outdated,
+ or stale AnalysisFindings.
+ Prioritizes stale entities (stale=true) over outdated version entities."
+ [entity-type batch-size]
+ (let [model (deps.dependency-types/dependency-type->model entity-type)
table-name (t2/table-name model)
- id-field (keyword (name table-name) "id")
- table-wildcard (keyword (name table-name) "*")]
+ id-field (keyword (name table-name) "id")
+ table-wildcard (keyword (name table-name) "*")]
(t2/select model
{:select [table-wildcard]
:from table-name
:left-join [:analysis_finding [:and
[:= :analysis_finding.analyzed_entity_id id-field]
- [:= :analysis_finding.analyzed_entity_type (name type)]]]
- :where [:<
- [:coalesce :analysis_finding.analysis_version 0]
- *current-analysis-finding-version*]
+ [:= :analysis_finding.analyzed_entity_type (name entity-type)]]]
+ :where [:or
+ [:= :analysis_finding.stale true] ; stale analysis
+ [:< ; missing or outdated analysis
+ [:coalesce :analysis_finding.analysis_version 0]
+ *current-analysis-finding-version*]]
+ ;; stale entities first
+ :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]]
:limit batch-size})))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
index 285502d04a4..0369b378945 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
@@ -1,6 +1,7 @@
(ns metabase-enterprise.dependencies.task.entity-check
(:require
[metabase-enterprise.dependencies.findings :as deps.findings]
+ [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
[metabase-enterprise.dependencies.settings :as deps.settings]
[metabase-enterprise.dependencies.task-util :as deps.task-util]
[metabase.premium-features.core :as premium-features]
@@ -9,19 +10,30 @@
(set! *warn-on-reflection* true)
+(defn- process-one-batch!
+ "Process one batch of entities. Returns true if the full batch was used."
+ []
+ (-> (reduce (fn [batch-size entity-type]
+ (if (< batch-size 1)
+ (reduced 0)
+ (let [processed (deps.findings/analyze-batch! entity-type batch-size)]
+ (when (pos? processed)
+ (log/info "Updated" processed "entities of type" entity-type))
+ (- batch-size processed))))
+ (deps.settings/dependency-entity-check-batch-size)
+ deps.findings/supported-entities)
+ (< 1)))
+
(defn- check-entities!
+ "Process entities, draining all stale entities before returning.
+ Returns true if the full batch was used (more work may remain)."
[]
(when (premium-features/has-feature? :dependencies)
- (-> (reduce (fn [batch-size entity-type]
- (if (< batch-size 1)
- (reduced 0)
- (let [processed (deps.findings/analyze-batch! entity-type batch-size)]
- (when (pos? processed)
- (log/info "Updated" processed "entities of type" entity-type))
- (- batch-size processed))))
- (deps.settings/dependency-entity-check-batch-size)
- deps.findings/supported-entities)
- (< 1))))
+ (loop []
+ (let [used-full-batch? (process-one-batch!)]
+ (if (and used-full-batch? (deps.analysis-finding/has-stale-entities?))
+ (recur)
+ used-full-batch?)))))
(declare schedule-next-run!)
@@ -56,3 +68,9 @@
deps.task-util/job-initial-delay
schedule-next-run!)
(log/info "Not starting dependency entity check job because the batch size is not positive")))
+
+(defn trigger-entity-check-job!
+ "Trigger the DependencyEntityCheck job to run immediately.
+ Use this when entities have been marked stale and need re-analysis."
+ []
+ (schedule-next-run! 0))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index 6199a1471df..c1f485570c3 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -18,15 +18,16 @@
(comment
metabase-enterprise.dependencies.events/keep-me)
-(defn- run-with-dependencies-setup [thunk]
+(defn- run-with-dependencies-setup! [thunk]
(mt/with-test-user :rasta
(mt/with-premium-features #{:dependencies}
- (lib-be/with-metadata-provider-cache
- (let [mp (mt/metadata-provider)]
- (thunk mp))))))
+ (mt/with-model-cleanup [:model/AnalysisFinding]
+ (lib-be/with-metadata-provider-cache
+ (let [mp (mt/metadata-provider)]
+ (thunk mp)))))))
(deftest dashboard-update-sets-correct-dependencies
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [products-id (mt/id :products)
orders-id (mt/id :orders)
@@ -143,7 +144,7 @@
(is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard))))))))))
(deftest document-update-sets-correct-dependencies
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
@@ -196,7 +197,7 @@
(deftest sandbox-update-sets-correct-dependencies
(mt/with-premium-features #{:sandboxes}
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "sandbox group"}
:model/Card {card1-id :id} {}
@@ -226,7 +227,7 @@
(deftest card-dependency-calculation-error-handling-test
(testing "When dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
@@ -254,7 +255,7 @@
(deftest snippet-dependency-calculation-error-handling-test
(testing "When snippet dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/NativeQuerySnippet {snippet-id :id :as snippet} {:name "test snippet"
:content "SELECT 1"}]
@@ -281,7 +282,7 @@
(deftest ^:sequential native-transform-updates-dependencies-test
(testing "native transform update events trigger dependency calculations"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/native-query mp "select * from orders")
:type :query}]
@@ -297,7 +298,7 @@
(deftest ^:sequential mbql-transform-updates-dependencies-test
(testing "mbql transform update events trigger dependency calculations"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
:type :query}]
@@ -313,7 +314,7 @@
(deftest ^:sequential python-transform-updates-dependencies-test
(testing "python transform update events trigger dependency calculations"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(let [source {:type :python,
:source-database 2,
@@ -335,7 +336,7 @@
(deftest ^:sequential transform-run-updates-dependencies-test
(testing "transform run events trigger dependency calculations"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
(mt/with-temp [:model/Transform {transform-id :id} {:target target}
@@ -356,7 +357,7 @@
(deftest ^:sequential python-transform-update-handles-downstream-dependencies-test
(testing "python transform update events handles downstream dependencies"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(let [source {:type :python,
:source-database (mt/id),
@@ -401,7 +402,7 @@
(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test
(testing "query transform update events handles downstream dependencies"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/native-query mp "select * from orders")
:type :query}
@@ -439,7 +440,7 @@
(deftest transform-dependency-calculation-error-handling-test
(testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/Transform {transform-id :id :as transform} {}]
(log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
@@ -465,7 +466,7 @@
(deftest dashboard-dependency-calculation-error-handling-test
(testing "When dashboard dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/Dashboard {dashboard-id :id :as dashboard} {}]
(log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
@@ -491,7 +492,7 @@
(deftest document-dependency-calculation-error-handling-test
(testing "When document dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/Document {document-id :id :as document} {:content_type "application/json+vnd.prose-mirror"
:document {:type "doc" :content []}}]
@@ -519,7 +520,7 @@
(deftest sandbox-dependency-calculation-error-handling-test
(testing "When sandbox dependency calculation throws an error, it should be logged and the version should still be updated"
(mt/with-premium-features #{:sandboxes}
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "test group"}
:model/Card {card-id :id} {}
@@ -548,7 +549,7 @@
(is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))))))
(deftest segment-update-sets-correct-dependencies
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)
@@ -579,7 +580,7 @@
(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))))
(deftest card-with-segment-dependencies
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)]
@@ -605,7 +606,7 @@
(deftest segment-dependency-calculation-error-handling-test
(testing "When segment dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [_]
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)]
@@ -633,7 +634,7 @@
(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))))))))
(deftest measure-update-sets-correct-dependencies
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(mt/with-test-user :rasta
(let [orders-id (mt/id :orders)
@@ -668,7 +669,7 @@
(is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))
(deftest measure-with-measure-dependencies
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [orders-id (mt/id :orders)
orders (lib.metadata/table mp orders-id)
@@ -697,7 +698,7 @@
(t2/select :model/Dependency :from_entity_id measure2-id :from_entity_type :measure)))))))))))))
(deftest card-with-measure-dependencies
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [orders-id (mt/id :orders)
orders (lib.metadata/table mp orders-id)
@@ -725,7 +726,7 @@
(deftest measure-dependency-calculation-error-handling-test
(testing "When measure dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(let [orders-id (mt/id :orders)
orders (lib.metadata/table mp orders-id)
@@ -774,9 +775,9 @@
(is (= expected-version (analyses-map id)))))))))
(deftest ^:sequential card-update-works-with-no-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
- (testing "Card updates should publish new analyses when no existing ones exist"
+ (testing "Card updates should analyze the card and mark dependents stale for async processing"
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
(mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
@@ -787,13 +788,16 @@
:to_entity_id parent-card-id}]
(events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
(assert-has-analyses
- {:card {parent-card-id -1
- child-card-id -1}})))))))
+ {:card {parent-card-id -1}})
+ (is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id child-card-id))
+ "entities without analysis findings should not be marked: no fake analysis record should be created")))))))
(deftest ^:sequential card-update-updates-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
- (testing "Card updates should update existing analyses when they exist"
+ (testing "Card updates should update the card's analysis and mark dependents stale"
(let [products-id (mt/id :products)
orders-id (mt/id :orders)
products (lib.metadata/table mp products-id)
@@ -815,16 +819,86 @@
{:card {parent-card-id old-version
child-card-id old-version
other-card-id old-version}}))
+ (t2/with-transaction [_conn]
+ (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))
+ (testing "Parent should be re-analyzed synchronously"
+ (assert-has-analyses
+ {:card {parent-card-id new-version}}))
+ (testing "Child should be marked stale (not re-analyzed yet)"
+ (let [child-finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id child-card-id)]
+ (is (= old-version (:analysis_version child-finding)))
+ (is (true? (:stale child-finding)))))
+ (testing "Other card should be unchanged"
+ (let [other-finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id other-card-id)]
+ (is (= old-version (:analysis_version other-finding)))
+ (is (false? (:stale other-finding))))))))))))
+
+(deftest ^:sequential stale-entities-are-processed-by-job-test
+ (run-with-dependencies-setup!
+ (fn [mp]
+ (testing "Stale entities are re-analyzed when the job runs"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ current-version models.analysis-finding/*current-analysis-finding-version*]
+ (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
+ :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id child-card-id
+ :to_entity_type :card
+ :to_entity_id parent-card-id}]
+ ;; Create initial analyses for both cards
+ (deps.findings/upsert-analysis! parent-card)
+ (deps.findings/upsert-analysis! child-card)
+ ;; Update parent - this marks child as stale
+ (t2/with-transaction [_conn]
+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ ;; Verify child is stale
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id child-card-id))
+ "Child should be marked stale after parent update"))
+ ;; Run the job to process stale entities
+ (deps.findings/analyze-batch! :card 10)
+ ;; Verify child is no longer stale and has been re-analyzed
+ (let [child-finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id child-card-id)]
+ (is (false? (:stale child-finding))
+ "Child should no longer be stale after job runs")
+ (is (= current-version (:analysis_version child-finding))
+ "Child should have current analysis version"))))))))
+
+(deftest ^:sequential card-update-transaction-rollback-test
+ (run-with-dependencies-setup!
+ (fn [mp]
+ (testing "If marking dependents stale fails, analysis upsert is rolled back"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ old-version models.analysis-finding/*current-analysis-finding-version*
+ new-version (inc old-version)]
+ (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]
+ ;; Create initial analysis
+ (deps.findings/upsert-analysis! card)
+ (testing "Initial analysis exists"
+ (is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id))))
+ ;; Try to update with mark-dependents-stale! throwing an exception
(binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))
- (testing "Checking that the analyses were updated appropriately"
- (assert-has-analyses
- {:card {parent-card-id new-version
- child-card-id new-version
- other-card-id old-version}}))))))))
+ (with-redefs [deps.findings/mark-dependents-stale! (fn [_ _] (throw (ex-info "Simulated failure" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated failure"
+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})))))
+ ;; Verify analysis was NOT updated (rolled back)
+ (testing "Analysis should be unchanged after failed event"
+ (is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id))))))))))
(deftest ^:sequential card-update-ignores-native-cards-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(testing "Native card updates should not trigger analysis"
(mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/native-query mp "select * from products")}
@@ -839,7 +913,7 @@
child-card-id nil}}))))))
(deftest ^:sequential card-update-stops-on-transforms-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(testing "Card updates should continue analyzing through a transform"
(let [products-id (mt/id :products)
@@ -873,15 +947,26 @@
child-card-id old-version}
:transform {transform-id old-version}}))
(binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))
- (testing "Checking that analyses were updated correctly"
- (assert-has-analyses
- {:card {parent-card-id new-version
- child-card-id old-version}
- :transform {transform-id new-version}}))))))))
+ (t2/with-transaction [_conn]
+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ (testing "parent card should be re-analyzed synchronously"
+ (is (=? {:analysis_version new-version :stale false}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id parent-card-id))))
+ (testing "transform (direct dependent) should be marked stale"
+ (is (=? {:analysis_version old-version :stale true}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :transform
+ :analyzed_entity_id transform-id))))
+ (testing "child card (transitive dependent) should be marked stale"
+ (is (=? {:analysis_version old-version :stale true}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id child-card-id))))))))))))
(deftest ^:sequential transform-update-works-with-no-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(testing "transform update creates new analyses"
(let [products-id (mt/id :products)
@@ -904,7 +989,7 @@
:transform {transform-id -1}})))))))
(deftest ^:sequential transform-update-updates-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(testing "transform update updates analyses when they exist"
(let [products-id (mt/id :products)
@@ -944,7 +1029,7 @@
:transform {transform-id new-version}})))))))))
(deftest ^:sequential transform-update-ignores-native-transforms-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(testing "Native transform updates should not trigger analysis"
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query
@@ -958,9 +1043,9 @@
{:transform {transform-id nil}}))))))
(deftest ^:sequential transform-run-works-with-no-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
- (testing "transform runs create analyses for dependent cards"
+ (testing "transform runs work when dependent cards have no pre-existing analyses"
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
@@ -974,25 +1059,23 @@
:from_entity_id card-id
:to_entity_type :transform
:to_entity_id transform-id}]
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "public"
- :output-table "sample"
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (assert-has-analyses
- {:card {card-id -1}})))))))
+ ;; Event should complete without error even when there are no pre-existing analyses
+ (is (some? (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "public"
+ :output-table "sample"
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})))))))))
(deftest ^:sequential transform-run-updates-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(testing "transform runs update existing analyses when they exist"
(let [products-id (mt/id :products)
orders-id (mt/id :orders)
products (lib.metadata/table mp products-id)
orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]
+ old-version models.analysis-finding/*current-analysis-finding-version*]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
:model/Transform {transform-id :id :as transform} {:source {:type :query
:query (lib/query mp products)}
@@ -1013,49 +1096,33 @@
{:card {card-id old-version
other-card-id old-version}
:transform {transform-id old-version}}))
- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
+ (t2/with-transaction [_conn]
(events/publish-event! :event/transform-run-complete
{:object {:db-id (mt/id)
:output-schema "public"
:output-table "sample"
:transform-id transform-id}
- :user-id api/*current-user-id*}))
- (testing "checking that analyses were updated appropriately"
- (assert-has-analyses
- {:card {card-id new-version
- other-card-id old-version}
- :transform {transform-id old-version}}))))))))
-
-(deftest ^:sequential transform-run-triggers-analysis-for-native-transforms-test
- (run-with-dependencies-setup
- (fn [mp]
- (testing "Native transform runs should still trigger analysis"
- (let [products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id} {:source {:type :query
- :query (lib/native-query mp "select * from products")}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :transform
- :to_entity_id transform-id}]
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "public"
- :output-table "sample"
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (assert-has-analyses
- {:card {card-id -1}})))))))
+ :user-id api/*current-user-id*})
+ (testing "dependent card should be marked stale"
+ (is (=? {:analysis_version old-version :stale true}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id))))
+ (testing "other card should be unchanged"
+ (is (=? {:analysis_version old-version :stale false}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id other-card-id))))
+ (testing "transform should be unchanged"
+ (is (=? {:analysis_version old-version :stale false}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :transform
+ :analyzed_entity_id transform-id)))))))))))
(deftest ^:sequential segment-update-works-with-no-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
- (testing "segment update creates analyses when necessary"
+ (testing "segment update creates analysis for the segment itself"
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)
products (lib.metadata/table mp products-id)]
@@ -1067,12 +1134,12 @@
:to_entity_type :segment
:to_entity_id segment-id}]
(events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})
+ ;; Segment should be analyzed synchronously
(assert-has-analyses
- {:card {card-id -1}
- :segment {segment-id -1}})))))))
+ {:segment {segment-id -1}})))))))
(deftest ^:sequential segment-update-updates-analyses-test
- (run-with-dependencies-setup
+ (run-with-dependencies-setup!
(fn [mp]
(testing "segment update updates analyses when they exist"
(let [products-id (mt/id :products)
@@ -1093,9 +1160,16 @@
(assert-has-analyses
{:card {card-id old-version}
:segment {segment-id old-version}}))
- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*}))
- (testing "checking that analyses were updated correctly"
- (assert-has-analyses
- {:card {card-id new-version}
- :segment {segment-id new-version}}))))))))
+ (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
+ (t2/with-transaction [_conn]
+ (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})
+ (testing "segment should be re-analyzed synchronously"
+ (is (=? {:analysis_version new-version :stale false}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :segment
+ :analyzed_entity_id segment-id))))
+ (testing "dependent card should be marked stale"
+ (is (=? {:analysis_version old-version :stale true}
+ (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id))))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
index 314c3971bb6..54ca5c7190c 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
@@ -3,11 +3,15 @@
[clojure.test :refer :all]
[metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[toucan2.core :as t2]))
+(use-fixtures :once (fixtures/initialize :db))
+
(defn- backfill-all-entity-analyses! []
(doseq [model [:card :transform :segment]]
(while (pos? (deps.findings/analyze-batch! model 100)))))
@@ -44,11 +48,16 @@
(deftest ^:sequential re-analyze-entities-when-analysis-version-bumped-test
(backfill-all-entity-analyses!)
- (mt/with-premium-features #{:dependencies}
- (is (= 0 (deps.findings/analyze-batch! :card 2)))
- (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
- ;; can't check the actual cards analyzed here because this could be any starting card in the data
- (is (= 2 (deps.findings/analyze-batch! :card 2))))))
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card _ {:dataset_query (lib/query mp products)}
+ :model/Card _ {:dataset_query (lib/query mp orders)}]
+ (is (= 2 (deps.findings/analyze-batch! :card 2)))
+ (is (= 0 (deps.findings/analyze-batch! :card 2)))
+ (binding [models.analysis-finding/*current-analysis-finding-version* (inc models.analysis-finding/*current-analysis-finding-version*)]
+ (is (= 2 (deps.findings/analyze-batch! :card 2))))))))
(deftest ^:sequential does-not-analyze-native-entities-test
(testing "assumes native queries are always fine"
@@ -62,3 +71,80 @@
:model/AnalysisFinding
:analyzed_entity_id card-id
:analyzed_entity_type :card))))))))
+
+(defn- stale-map
+ "Returns a map of {entity-id stale?} for the given card IDs."
+ [card-ids]
+ (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id [:in card-ids]))
+
+(deftest ^:sequential analyze-batch-picks-up-missing-analyses-test
+ (testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFinding]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
+ (testing "card has no analysis finding initially"
+ (is (not (t2/exists? :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id))))
+ (testing "analyze-batch! creates analysis for the card"
+ (lib-be/with-metadata-provider-cache
+ (is (pos? (deps.findings/analyze-batch! :card 10)))))
+ (testing "card now has an analysis finding"
+ (is (t2/exists? :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)))))))))
+
+(deftest mark-dependents-stale-test
+ (testing "mark-dependents-stale! marks direct dependents as stale"
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-premium-features #{:dependencies}
+ (lib-be/with-metadata-provider-cache
+ (mt/with-model-cleanup [:model/AnalysisFinding]
+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id child-card-id
+ :to_entity_type :card
+ :to_entity_id parent-card-id}]
+ (run! deps.findings/upsert-analysis!
+ (t2/select :model/Card :id [:in [parent-card-id child-card-id]]))
+ (is (= {parent-card-id false, child-card-id false}
+ (stale-map [parent-card-id child-card-id]))
+ "neither should be stale before marking")
+ (t2/with-transaction [_conn]
+ (deps.findings/mark-dependents-stale! :card parent-card-id)
+ (is (= {parent-card-id false, child-card-id true}
+ (stale-map [parent-card-id child-card-id]))
+ "parent should NOT be stale, child should be stale")))))))))
+
+(deftest mark-dependents-stale-transitive-test
+ (testing "mark-dependents-stale! marks transitive dependents as stale"
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-premium-features #{:dependencies}
+ (lib-be/with-metadata-provider-cache
+ (mt/with-model-cleanup [:model/AnalysisFinding]
+ (mt/with-temp [:model/Card {grandparent-id :id} {:dataset_query (lib/query mp products)}
+ :model/Card {parent-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp grandparent-id))}
+ :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id parent-id
+ :to_entity_type :card
+ :to_entity_id grandparent-id}
+ :model/Dependency _ {:from_entity_type :card
+ :from_entity_id child-id
+ :to_entity_type :card
+ :to_entity_id parent-id}]
+ (run! deps.findings/upsert-analysis!
+ (t2/select :model/Card :id [:in [grandparent-id parent-id child-id]]))
+ (t2/with-transaction [_conn]
+ (deps.findings/mark-dependents-stale! :card grandparent-id)
+ (is (= {grandparent-id false, parent-id true, child-id true}
+ (stale-map [grandparent-id parent-id child-id]))
+ "grandparent should NOT be stale, parent and child should be stale (transitive)")))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
index 09cd1929ed2..f7010d0e585 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
@@ -356,28 +356,33 @@
(jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis
- ;; (because analyzed_at < field.updated_at) and trigger re-analysis.
+ ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.
(sync/sync-database! (mt/db))
;; Verify the field is now inactive
(testing "Field is marked inactive after sync"
(is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))
- ;; Step 6 & 7: The sync-end event should have triggered re-analysis.
- ;; Check that analysis now fails with missing column error.
- (testing "Re-analysis detects the missing column"
- (let [finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- (is (some? finding) "Analysis finding should exist")
- (is (false? (:result finding)) "Analysis should fail")
- (is (not= initial-analysis-time (:analyzed_at finding))
- "Analysis should have been re-run"))
-
- (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]
- (is (seq errors) "Should have analysis errors")
- (is (some #(= :missing-column (:error_type %)) errors)
- "Should have a missing-column error"))))))))
+ ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.
+ ;; Due to race conditions, the entity may be either stale (job hasn't run yet)
+ ;; or already reanalyzed (job ran). Check for either valid outcome.
+ (let [finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ (is (some? finding) "Analysis finding should exist")
+ (if (:stale finding)
+ ;; Job hasn't run yet - sync marked it stale, that's the expected behavior
+ (testing "Entity is marked stale for re-analysis"
+ (is (true? (:stale finding))))
+ ;; Job already ran - verify the reanalysis detected the missing column
+ (testing "Re-analysis detects the missing column"
+ (is (false? (:result finding)) "Analysis should fail")
+ (is (not= initial-analysis-time (:analyzed_at finding))
+ "Analysis should have been re-run")
+ (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]
+ (is (seq errors) "Should have analysis errors")
+ (is (some #(= :missing-column (:error_type %)) errors)
+ "Should have a missing-column error"))))))))))
;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all
;; cards which depend on that table.
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
new file mode 100644
index 00000000000..4f5e0759860
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
@@ -0,0 +1,44 @@
+(ns metabase-enterprise.dependencies.models.analysis-finding-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest upsert-analysis-clears-stale-flag-test
+ (testing "upsert-analysis! sets stale to false"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {}]
+ (t2/with-transaction [_conn]
+ ;; Insert a finding with stale = true
+ (t2/insert! :model/AnalysisFinding
+ {:analyzed_entity_type :card
+ :analyzed_entity_id card-id
+ :analysis_version (dec models.analysis-finding/*current-analysis-finding-version*)
+ :analyzed_at (java.time.OffsetDateTime/now)
+ :result true
+ :stale true})
+ ;; Verify it's stale
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)))
+ ;; Now upsert a new analysis
+ (models.analysis-finding/upsert-analysis! :card card-id true [])
+ ;; Verify stale is now false
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id))))))))
+
+(deftest upsert-analysis-new-finding-has-stale-false-test
+ (testing "upsert-analysis! creates new findings with stale = false"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {}]
+ (t2/with-transaction [_conn]
+ ;; Upsert a new analysis (no existing finding)
+ (models.analysis-finding/upsert-analysis! :card card-id true [])
+ ;; Verify stale is false
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id))))))))
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index 2858201e529..6471e7af8a0 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -804,6 +804,42 @@ databaseChangeLog:
tableName: python_library
constraintName: unique_python_library_entity_id
+ - changeSet:
+ id: v59.2026-01-28T10:00:00
+ author: metamben
+ comment: Add stale column to analysis_finding for asynchronous dependent updates
+ changes:
+ - addColumn:
+ tableName: analysis_finding
+ columns:
+ - column:
+ name: stale
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the entity needs re-analysis due to upstream changes
+ constraints:
+ nullable: false
+ rollback:
+ - dropColumn:
+ tableName: analysis_finding
+ columnName: stale
+
+ - changeSet:
+ id: v59.2026-01-28T10:00:01
+ author: metamben
+ comment: Add partial index on analysis_finding.stale for efficient polling
+ changes:
+ - sql:
+ dbms: postgresql
+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale) WHERE stale = true
+ - sql:
+ dbms: mysql,mariadb,h2
+ sql: CREATE INDEX idx_analysis_finding_stale ON analysis_finding(stale)
+ rollback:
+ - dropIndex:
+ tableName: analysis_finding
+ indexName: idx_analysis_finding_stale
+
# >>>>>>>>>> DO NOT ADD NEW MIGRATIONS BELOW THIS LINE! ADD THEM ABOVE <<<<<<<<<<
########################################################################################################################