The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GHY-3232: Make dependency graph calculation async on entity updates (#71037)
Eric Normand5d1d7d4a49bd
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 912c61dcb8b..1e993e18095 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2843,7 +2843,6 @@apiapp-dbcollections- configdocumentsdriverevents@@ -2868,8 +2867,6 @@:model-imports #{:model/Card:model/Collection:model/Dashboard- :model/DashboardCard- :model/DashboardCardSeries:model/Document:model/Field:model/Measurediff --git a/e2e/support/helpers/e2e-dependency-helpers.ts b/e2e/support/helpers/e2e-dependency-helpers.tsindex cba03fcb949..04f39db9b80 100644--- a/e2e/support/helpers/e2e-dependency-helpers.ts+++ b/e2e/support/helpers/e2e-dependency-helpers.ts@@ -1,5 +1,8 @@import type {+ DependencyGraph as DependencyGraphData,+ DependencyId,DependencyNode,+ DependencyType,ListBreakingGraphNodesResponse,ListUnreferencedGraphNodesResponse,} from "metabase-types/api";@@ -42,9 +45,56 @@ export const DependencyDiagnostics = {},};-const WAIT_TIMEOUT = 10000;+const WAIT_TIMEOUT = 30000;const WAIT_INTERVAL = 100;+export function waitForBackfillComplete(+ timeout = WAIT_TIMEOUT,+): Cypress.Chainable {+ return cy+ .request<{+ complete: boolean;+ }>("GET", "/api/ee/dependencies/backfill-status")+ .then((response) => {+ if (response.body.complete) {+ return cy.wrap(response);+ } else if (timeout > 0) {+ cy.wait(WAIT_INTERVAL);+ return waitForBackfillComplete(timeout - WAIT_INTERVAL);+ } else {+ throw new Error("Dependency backfill timeout");+ }+ });+}++export function waitForGraphDependencies(+ id: DependencyId,+ type: DependencyType,+ filter: (graph: DependencyGraphData) => boolean,+ timeout = WAIT_TIMEOUT,+): Cypress.Chainable {+ return cy+ .request<DependencyGraphData>(+ "GET",+ `/api/ee/dependencies/graph?id=${id}&type=${type}`,+ )+ .then((response) => {+ if (filter(response.body)) {+ return cy.wrap(response);+ } else if (timeout > 0) {+ cy.wait(WAIT_INTERVAL);+ return waitForGraphDependencies(+ id,+ type,+ filter,+ timeout - WAIT_INTERVAL,+ );+ } else {+ throw new Error("Dependency graph retry timeout");+ }+ });+}+export function waitForUnreferencedEntities(filter: (nodes: DependencyNode[]) => boolean,timeout = WAIT_TIMEOUT,diff --git a/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.ts b/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.tsindex 8d6568cc73a..88e89d2a0ce 100644--- a/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.ts@@ -102,6 +102,8 @@ describe("scenarios > data studio > library > metrics", () => {H.echartsContainer().findByText("Count").should("be.visible");cy.log("Verify metric dependencies page");+ H.waitForBackfillComplete();+ cy.reload();H.DataStudio.Metrics.dependenciesTab().click();H.DependencyGraph.graph().within(() => {cy.findByText("Orders").should("be.visible");diff --git a/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.ts b/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.tsindex 3dcb4c71ab8..ffa3721f34b 100644--- a/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.ts@@ -160,7 +160,7 @@ describe("scenarios > data studio > library > tables", () => {name: "Test question",query: { "source-table": ORDERS_ID },});-+ H.waitForBackfillComplete();H.DataStudio.Tables.visitOverviewPage(ORDERS_ID);H.DataStudio.Tables.dependenciesTab().click();H.DependencyGraph.graph().within(() => {diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.tsindex 7b93e9952db..dcc33bd244e 100644--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts@@ -1924,6 +1924,7 @@ LIMITvisitTransform: true,});+ H.waitForBackfillComplete();H.DataStudio.Transforms.dependenciesTab().click();H.DataStudio.Dependencies.content().should("contain", "Transform B")diff --git a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.tsindex 617db43985f..2f2d69182bd 100644--- a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts+++ b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts@@ -690,6 +690,8 @@ function visitGraph() {}function visitGraphForEntity(id: DependencyId, type: DependencyType) {+ // Wait for async dependency computation to complete before navigating+ H.waitForBackfillComplete();return cy.visit(BASE_URL, { qs: { id, type } });}diff --git a/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.tsindex 10db442122b..813854fa17a 100644--- a/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.ts+++ b/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.ts@@ -120,6 +120,7 @@ describe("scenarios > dependencies > unreferenced list", () => {describe("analysis", () => {it("should show unreferenced entities", () => {setupEntities();+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();H.DependencyDiagnostics.list().within(() => {ENTITY_NAMES.forEach((name) => {@@ -142,6 +143,7 @@ describe("scenarios > dependencies > unreferenced list", () => {describe("search", () => {it("should search for entities", () => {setupEntities();+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();H.DependencyDiagnostics.searchInput().type(MODEL_FOR_QUESTION_DATA_SOURCE,@@ -154,6 +156,7 @@ describe("scenarios > dependencies > unreferenced list", () => {it("should search for entities with type filters", () => {setupEntities();+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();H.DependencyDiagnostics.searchInput().type("tag");checkList({@@ -176,6 +179,7 @@ describe("scenarios > dependencies > unreferenced list", () => {describe("filters", () => {it("should filter entities by type", () => {setupEntities();+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();checkList({ visibleEntities: ENTITY_NAMES });@@ -206,9 +210,7 @@ describe("scenarios > dependencies > unreferenced list", () => {it("should persist filter changes after page reload", () => {setupEntities();- H.waitForUnreferencedEntities((entities) =>- entities.some((e) => e.data.name === MODEL_FOR_QUESTION_DATA_SOURCE),- );+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();checkList({ visibleEntities: MODEL_NAMES });@@ -222,9 +224,7 @@ describe("scenarios > dependencies > unreferenced list", () => {it("should filter by location", () => {setupEntities();- H.waitForUnreferencedEntities((entities) =>- entities.some((e) => e.data.name === MODEL_FOR_MODEL_DATA_SOURCE),- );+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();checkList({visibleEntities: [@@ -258,6 +258,7 @@ describe("scenarios > dependencies > unreferenced list", () => {describe("sorting", () => {it("should sort by name", () => {setupEntities();+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();H.DependencyDiagnostics.searchInput().type("Model for");@@ -281,6 +282,7 @@ describe("scenarios > dependencies > unreferenced list", () => {it("should sort by location", () => {setupEntities();+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();H.DependencyDiagnostics.searchInput().type("Model for");@@ -299,6 +301,7 @@ describe("scenarios > dependencies > unreferenced list", () => {it("should persist sorting changes after page reload", () => {setupEntities();+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();H.DependencyDiagnostics.searchInput().type("Model for");@@ -314,12 +317,7 @@ describe("scenarios > dependencies > unreferenced list", () => {describe("selecting entities", () => {it("should show the sidebar for supported entities and trigger snowplow event", () => {setupEntities();- H.waitForUnreferencedEntities((entities) =>- entities.some(- (e) =>- e.type === "table" && e.data.display_name === TABLE_DISPLAY_NAME,- ),- );+ H.waitForBackfillComplete();H.DependencyDiagnostics.visitUnreferencedEntities();H.DependencyDiagnostics.list().findByText(TABLE_DISPLAY_NAME).click();diff --git a/e2e/test/scenarios/metrics/metric-page.cy.spec.ts b/e2e/test/scenarios/metrics/metric-page.cy.spec.tsindex 159b06074b0..9b3fb2443a7 100644--- a/e2e/test/scenarios/metrics/metric-page.cy.spec.ts+++ b/e2e/test/scenarios/metrics/metric-page.cy.spec.ts@@ -376,6 +376,7 @@ describe("scenarios > metrics > metric page", () => {it("should show the Dependencies tab with dependency graph in EE", () => {H.createQuestion(ORDERS_SCALAR_METRIC).then(({ body: metric }) => {+ H.waitForBackfillComplete();H.visitMetric(metric.id);});diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex 4c1d35f95a0..7c4e295f690 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -7,6 +7,7 @@[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types][metabase-enterprise.dependencies.models.analysis-finding-error :as analysis-finding-error][metabase-enterprise.dependencies.models.dependency :as dependency]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status][metabase.analyze.core :as analyze][metabase.api.common :as api][metabase.api.macros :as api.macros]@@ -1187,6 +1188,13 @@(fetch-and-hydrate-nodes nodes-by-type))(sort-dependents sort-column sort-direction)))))+(api.macros/defendpoint :get "/backfill-status" :- [:map+ [:complete :boolean]]+ "Returns whether the dependency backfill has pending work.+ `complete` is true when there are no stale or outdated entities awaiting processing."+ [_route-params _query-params]+ {:complete (not (deps.dependency-status/has-stale-or-outdated?))})+(def ^{:arglists '([request respond raise])} routes"`/api/ee/dependencies` routes."(handlers/routesdiff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex 41260a6060a..03fa2bbac71 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -6,12 +6,26 @@[metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema]- [metabase.queries.schema :as queries.schema][metabase.transforms-base.util :as transforms-base.u][metabase.util :as u][metabase.util.log :as log][metabase.util.malli :as mu]))+(defmulti calculate-deps*+ "Implementation multimethod for [[calculate-deps]]. Dispatches on entity-type keyword.+ Prefer calling [[calculate-deps]] which validates the return value."+ {:arglists '([entity-type entity])}+ (fn [entity-type _entity] entity-type))++(mu/defn calculate-deps :- ::deps.schema/upstream-deps+ "Calculate upstream dependencies for a single entity.+ Returns a map of dependency-type -> set of entity IDs."+ [entity-type :- keyword?+ entity]+ (calculate-deps* entity-type entity))++;;; ------------------------------------------------ Helpers ------------------------------------------------+(mu/defn- upstream-deps:mbql-query :- ::deps.schema/upstream-deps[query :- ::lib.schema/query]{:card (or (lib/all-source-card-ids query) #{})@@ -34,9 +48,32 @@(upstream-deps:native-query query)(upstream-deps:mbql-query query)))-(mu/defn upstream-deps:card :- ::deps.schema/upstream-deps- "Given a Toucan `:model/Card`, return its upstream dependencies as a map from the kind to a set of IDs."- [{query :dataset_query :as card} :- ::queries.schema/card]+(mu/defn- upstream-deps:python-transform :- ::deps.schema/upstream-deps+ [{{tables :source-tables} :source :as _py-transform}+ :- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]+ {:table (into #{} (keep :table_id) tables)})++;; Modified implementation of documents.models.document/document-deps+(defn- document-deps+ [{:keys [content_type] :as document}]+ (when (= content_type prose-mirror/prose-mirror-content-type)+ (prose-mirror/collect-ast document (fn [{:keys [type attrs]}]+ (cond+ (and (= prose-mirror/smart-link-type type)+ (#{"card" "dashboard" "table" "document"} (:model attrs)))+ [(keyword (:model attrs)) (:entityId attrs)]++ (= prose-mirror/card-embed-type type)+ [:card (:id attrs)]++ :else+ nil)))))++;;; ------------------------------------------------ defmethods ------------------------------------------------++(defmethod calculate-deps* :card+ [_ {query :dataset_query :as card}]+ {:pre [(some? query)]}(let [query-deps (upstream-deps:query query)param-card-ids (keep #(-> % :values_source_config :card_id) (:parameters card))](reduce (fn [deps card-id]@@ -44,21 +81,8 @@query-depsparam-card-ids)))-(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps- "Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."- [{{tables :source-tables} :source :as _py-transform}- :- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]- {:table (into #{} (keep :table_id) tables)})--(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps- "Given a Transform (in Toucan form), return its upstream dependencies."- [{{:keys [query]} :source :as transform} :-- [:map- [:source [:multi {:dispatch (comp keyword :type)}- [:query- [:map [:query ::lib.schema/query]]]- [:python- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]]]]]+(defmethod calculate-deps* :transform+ [_ {{:keys [query]} :source :as transform}](let [source-type (transforms-base.u/transform-type transform)](case source-type:query (upstream-deps:query query)@@ -66,9 +90,8 @@(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type){}))))-(mu/defn upstream-deps:snippet :- ::deps.schema/upstream-deps- "Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."- [{:keys [template_tags] :as _snippet}]+(defmethod calculate-deps* :snippet+ [_ {:keys [template_tags] :as _snippet}](let [type->id-key {:card :card-id, :snippet :snippet-id}dependencies (keep (fn [tag](let [entity-type (:type tag)]@@ -78,10 +101,10 @@(vals template_tags))](u/group-by first second conj #{} dependencies)))-(mu/defn upstream-deps:dashboard :- ::deps.schema/upstream-deps- "Given a dashboard, return its upstream dependencies"- [{:keys [dashcards series-card-ids] :as dashboard}]+(defmethod calculate-deps* :dashboard+ [_ {:keys [dashcards] :as dashboard}](let [card-ids (into #{} (keep :card_id dashcards))+ series-card-ids (into #{} (comp (mapcat :series) (map :id)) dashcards)param-card-ids (into #{} (keep (comp :card_id :values_source_config) (:parameters dashboard)))vis-setting-target-ids (fn [link-type](into #{} (keep (fn [dashcard]@@ -111,47 +134,27 @@{:card all-card-ids:dashboard all-dashboard-ids}))-;; Modified implementation of documents.models.document/document-deps-(defn- document-deps- [{:keys [content_type] :as document}]- (when (= content_type prose-mirror/prose-mirror-content-type)- (prose-mirror/collect-ast document (fn [{:keys [type attrs]}]- (cond- (and (= prose-mirror/smart-link-type type)- (#{"card" "dashboard" "table" "document"} (:model attrs)))- [(keyword (:model attrs)) (:entityId attrs)]-- (= prose-mirror/card-embed-type type)- [:card (:id attrs)]-- :else- nil)))))--(mu/defn upstream-deps:document :- ::deps.schema/upstream-deps- "Given a document, return its upstream dependencies"- [document]+(defmethod calculate-deps* :document+ [_ document](reduce (fn [deps [dep-type dep-id]](update deps dep-type (fnil conj #{}) dep-id)){}(document-deps document)))-(mu/defn upstream-deps:sandbox :- ::deps.schema/upstream-deps- "Given a sandbox, return its upstream dependencies"- [sandbox]+(defmethod calculate-deps* :sandbox+ [_ sandbox](if-let [card-id (:card_id sandbox)]{:card #{card-id}}{}))-(mu/defn upstream-deps:segment :- ::deps.schema/upstream-deps- "Given a segment, return its upstream dependencies (the table it filters and any segments it references)"- [{:keys [table_id definition] :as _segment}]+(defmethod calculate-deps* :segment+ [_ {:keys [table_id definition] :as _segment}]{:segment (or (lib/all-segment-ids definition) #{}):table (cond-> (into #{} (lib/all-implicitly-joined-table-ids definition))table_id (conj table_id))})-(mu/defn upstream-deps:measure :- ::deps.schema/upstream-deps- "Given a measure, return its upstream dependencies (the table it aggregates, any measures it references, and any segments it references)"- [{:keys [table_id definition] :as _measure}]+(defmethod calculate-deps* :measure+ [_ {:keys [table_id definition] :as _measure}]{:measure (or (lib/all-measure-ids definition) #{}):segment (or (lib/all-segment-ids definition) #{}):table (cond-> (into #{} (lib/all-implicitly-joined-table-ids definition))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.cljindex 8f38a363ef0..98a90d90b9b 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj@@ -1,14 +1,12 @@(ns metabase-enterprise.dependencies.events(:require- [metabase-enterprise.dependencies.async :as async]- [metabase-enterprise.dependencies.calculation :as deps.calculation][metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]+ [metabase-enterprise.dependencies.task.backfill :as task.backfill][metabase-enterprise.dependencies.task.entity-check :as task.entity-check][metabase.events.core :as events]- [metabase.lib-be.core :as lib-be][metabase.premium-features.core :as premium-features]- [metabase.transforms.core :as transforms][metabase.util.log :as log][methodical.core :as methodical][toucan2.core :as t2]))@@ -17,42 +15,27 @@;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created;; or changed at that point.--(defmacro ignore-errors- "Ignore errors.-- In practice, we cannot reliably distinguish permanent and temporary errors, so in principle- every error should be retried a few times. Unfortunately that doesn't work, because updating the- dependency_analysis_version field itself is a trigger for new analysis, so the caller has no- way to give up and commit the new version after a few retries. We stop on every error until this- becomes a serious enough issue, at which point we will have to redesign version marking and- analysis triggering."- {:style/indent 0}- [entity-type entity-id & body]- `(try- ~@body- (catch Throwable e#- (log/error e# "Dependency calculation failed" {:entity-type ~entity-type- :entity-id ~entity-id})- nil)))+;;+;; Create/update handlers mark entities stale in dependency_status. The backfill task does the actual computation.++(defn- mark-stale-and-trigger!+ "Mark an entity as stale in dependency_status and trigger the backfill job."+ [entity-type entity-id]+ (try+ (deps.dependency-status/mark-stale! entity-type [entity-id])+ (task.backfill/trigger-backfill-job!)+ (catch Throwable e+ (log/error e "Failed to mark entity stale" {:entity-type entity-type :entity-id entity-id}))));; ### Cards(derive ::card-deps :metabase/event)(derive :event/card-create ::card-deps)(derive :event/card-update ::card-deps)-;; Backfill-only event that triggers dependency calculation without creating a revision (#66365)-(derive :event/card-dependency-backfill ::card-deps)(methodical/defmethod events/publish-event! ::card-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (models.dependency/replace-dependencies! :card (:id object)- (ignore-errors :card (:id object)- (deps.calculation/upstream-deps:card object)))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/Card (:id object)- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))+ (mark-stale-and-trigger! :card (:id object))))(derive ::card-delete :metabase/event)(derive :event/card-delete ::card-delete)@@ -60,7 +43,8 @@(methodical/defmethod events/publish-event! ::card-delete[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))))+ (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))+ (t2/delete! :model/DependencyStatus :entity_type :card :entity_id (:id object))));; ### Snippets(derive ::snippet-deps :metabase/event)@@ -70,13 +54,7 @@(methodical/defmethod events/publish-event! ::snippet-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (models.dependency/replace-dependencies! :snippet (:id object)- (ignore-errors :snippet (:id object)- (deps.calculation/upstream-deps:snippet object)))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/NativeQuerySnippet (:id object)- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))+ (mark-stale-and-trigger! :snippet (:id object))))(derive ::snippet-delete :metabase/event)(derive :event/snippet-delete ::snippet-delete)@@ -84,48 +62,18 @@(methodical/defmethod events/publish-event! ::snippet-delete[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))))+ (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))+ (t2/delete! :model/DependencyStatus :entity_type :snippet :entity_id (:id object))));; ### Transforms(derive ::transform-deps :metabase/event)(derive :event/create-transform ::transform-deps)(derive :event/update-transform ::transform-deps)-;; On *saving* a transform, the upstream deps of its query are computed and saved.-(defn- drop-outdated-target-dep! [{:keys [id target] :as transform}]- (let [db-id (transforms/transform-source-database transform)- downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency- :from_entity_type :table- :to_entity_type :transform- :to_entity_id id)- downstream-tables (when (seq downstream-table-ids)- (t2/select :model/Table :id [:in downstream-table-ids]))- outdated-tables (remove (fn [table]- (and (= (:schema table) (:schema target))- (= (:name table) (:name target))- (or (not db-id)- (= db-id (:db_id table)))))- downstream-tables)- not-found-table-ids (remove (into #{} (map :id) downstream-tables)- downstream-table-ids)]- (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)- (map :id) outdated-tables))]- (t2/delete! :model/Dependency- :from_entity_type :table- :from_entity_id [:in outdated-downstream-table-ids]- :to_entity_type :transform- :to_entity_id id))))-(methodical/defmethod events/publish-event! ::transform-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (models.dependency/replace-dependencies! :transform (:id object)- (ignore-errors :transform (:id object)- (deps.calculation/upstream-deps:transform object)))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/Transform (:id object) {:dependency_analysis_version models.dependency/current-dependency-analysis-version}))- (drop-outdated-target-dep! object))))+ (mark-stale-and-trigger! :transform (:id object))))(derive ::transform-delete :metabase/event)(derive :event/delete-transform ::transform-delete)@@ -134,7 +82,8 @@[_ {:keys [id]}];; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?(when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)))+ (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)+ (t2/delete! :model/DependencyStatus :entity_type :transform :entity_id id)));; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.;; (And if the target has changed, the old table's dep on the transform is dropped.)@@ -157,25 +106,11 @@(derive ::dashboard-deps :metabase/event)(derive :event/dashboard-create ::dashboard-deps)(derive :event/dashboard-update ::dashboard-deps)-;; Backfill-only event that triggers dependency calculation without creating a revision-(derive :event/dashboard-dependency-backfill ::dashboard-deps)(methodical/defmethod events/publish-event! ::dashboard-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (let [dashboard-id (:id object)- dashcards (t2/select :model/DashboardCard :dashboard_id dashboard-id)- series-card-ids (when (seq dashcards)- (t2/select-fn-set :card_id :model/DashboardCardSeries- :dashboardcard_id [:in (map :id dashcards)]))- dashboard (assoc object :dashcards dashcards :series-card-ids series-card-ids)]- (models.dependency/replace-dependencies! :dashboard dashboard-id- (ignore-errors :dashboard dashboard-id- (deps.calculation/upstream-deps:dashboard dashboard))))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/Dashboard (:id object)- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))+ (mark-stale-and-trigger! :dashboard (:id object))))(derive ::dashboard-delete :metabase/event)(derive :event/dashboard-delete ::dashboard-delete)@@ -183,7 +118,8 @@(methodical/defmethod events/publish-event! ::dashboard-delete[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :dashboard :from_entity_id (:id object))))+ (t2/delete! :model/Dependency :from_entity_type :dashboard :from_entity_id (:id object))+ (t2/delete! :model/DependencyStatus :entity_type :dashboard :entity_id (:id object))));; ### Documents(derive ::document-deps :metabase/event)@@ -193,13 +129,7 @@(methodical/defmethod events/publish-event! ::document-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (models.dependency/replace-dependencies! :document (:id object)- (ignore-errors :document (:id object)- (deps.calculation/upstream-deps:document object)))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/Document (:id object)- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))+ (mark-stale-and-trigger! :document (:id object))))(derive ::document-delete :metabase/event)(derive :event/document-delete ::document-delete)@@ -207,7 +137,8 @@(methodical/defmethod events/publish-event! ::document-delete[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :document :from_entity_id (:id object))))+ (t2/delete! :model/Dependency :from_entity_type :document :from_entity_id (:id object))+ (t2/delete! :model/DependencyStatus :entity_type :document :entity_id (:id object))));; ### Sandboxes(derive ::sandbox-deps :metabase/event)@@ -217,12 +148,7 @@(methodical/defmethod events/publish-event! ::sandbox-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (models.dependency/replace-dependencies! :sandbox (:id object) (ignore-errors :sandbox (:id object)- (deps.calculation/upstream-deps:sandbox object)))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/Sandbox (:id object)- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))+ (mark-stale-and-trigger! :sandbox (:id object))))(derive ::sandbox-delete :metabase/event)(derive :event/sandbox-delete ::sandbox-delete)@@ -230,7 +156,8 @@(methodical/defmethod events/publish-event! ::sandbox-delete[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :sandbox :from_entity_id (:id object))))+ (t2/delete! :model/Dependency :from_entity_type :sandbox :from_entity_id (:id object))+ (t2/delete! :model/DependencyStatus :entity_type :sandbox :entity_id (:id object))));; ### Segments(derive ::segment-deps :metabase/event)@@ -240,20 +167,16 @@(methodical/defmethod events/publish-event! ::segment-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (models.dependency/replace-dependencies! :segment (:id object)- (ignore-errors :segment (:id object)- (deps.calculation/upstream-deps:segment object)))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/Segment (:id object)- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))+ (mark-stale-and-trigger! :segment (:id object))))(derive ::segment-delete :metabase/event)(derive :event/segment-delete ::segment-delete)(methodical/defmethod events/publish-event! ::segment-delete[_ {:keys [object]}]- (t2/delete! :model/Dependency :from_entity_type :segment :from_entity_id (:id object)))+ (when (premium-features/has-feature? :dependencies)+ (t2/delete! :model/Dependency :from_entity_type :segment :from_entity_id (:id object))+ (t2/delete! :model/DependencyStatus :entity_type :segment :entity_id (:id object))));; ### Measures(derive ::measure-deps :metabase/event)@@ -263,13 +186,7 @@(methodical/defmethod events/publish-event! ::measure-deps[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/with-transaction [_conn]- (models.dependency/replace-dependencies! :measure (:id object)- (ignore-errors :measure (:id object)- (deps.calculation/upstream-deps:measure object)))- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)- (t2/update! :model/Measure (:id object)- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))+ (mark-stale-and-trigger! :measure (:id object))))(derive ::measure-delete :metabase/event)(derive :event/measure-delete ::measure-delete)@@ -277,50 +194,76 @@(methodical/defmethod events/publish-event! ::measure-delete[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))))+ (t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))+ (t2/delete! :model/DependencyStatus :entity_type :measure :entity_id (:id object))))++;; ## Checking dependents for breakage (analysis_finding staleness)+;;+;; This is a SEPARATE staleness system from the dependency_status table above.+;; - dependency_status.stale: tracks whether an entity's upstream *dependency graph* needs recomputation.+;; Handled by the backfill job (task.backfill).+;; - analysis_finding staleness: tracks whether an entity's dependents need re-analysis for *breakage detection*+;; (e.g., a model column was removed — are downstream cards broken?). Handled by the entity-check job+;; (task.entity-check).+;;+;; Both are triggered from the same entity events but serve different purposes and run independently.(derive ::check-card-dependents :metabase/event)(derive :event/card-create ::check-card-dependents)(derive :event/card-update ::check-card-dependents)-(derive :event/card-delete ::check-card-dependents)(methodical/defmethod events/publish-event! ::check-card-dependents[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (async/submit!- (fn []- (lib-be/with-metadata-provider-cache- (let [has-stale-dependents? (t2/with-transaction [_conn]- (deps.findings/upsert-analysis! object)- (deps.findings/mark-dependents-stale! :card (:id object)))]- (when has-stale-dependents?- (task.entity-check/trigger-entity-check-job!))))))))+ (deps.findings/mark-entity-stale! :card (:id object))+ (task.entity-check/trigger-entity-check-job!)))++(derive ::check-card-dependents-on-delete :metabase/event)+(derive :event/card-delete ::check-card-dependents-on-delete)++(methodical/defmethod events/publish-event! ::check-card-dependents-on-delete+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (when (deps.findings/mark-immediate-dependents-stale! :card (:id object))+ (task.entity-check/trigger-entity-check-job!))))(derive ::check-transform :metabase/event)(derive :event/create-transform ::check-transform)(derive :event/update-transform ::check-transform)-(derive :event/delete-transform ::check-transform)(methodical/defmethod events/publish-event! ::check-transform[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! object))))+ (deps.findings/mark-entity-stale! :transform (:id object))+ (task.entity-check/trigger-entity-check-job!)))++(derive ::check-transform-on-delete :metabase/event)+(derive :event/delete-transform ::check-transform-on-delete)++(methodical/defmethod events/publish-event! ::check-transform-on-delete+ [_ {:keys [id]}]+ (when (premium-features/has-feature? :dependencies)+ (when (deps.findings/mark-immediate-dependents-stale! :transform id)+ (task.entity-check/trigger-entity-check-job!))))(derive ::check-segment-dependents :metabase/event)(derive :event/segment-create ::check-segment-dependents)(derive :event/segment-update ::check-segment-dependents)-(derive :event/segment-delete ::check-segment-dependents)(methodical/defmethod events/publish-event! ::check-segment-dependents[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (lib-be/with-metadata-provider-cache- (let [has-stale-dependents? (t2/with-transaction [_conn]- (deps.findings/upsert-analysis! object)- (deps.findings/mark-dependents-stale! :segment (:id object)))]- (when has-stale-dependents?- (task.entity-check/trigger-entity-check-job!))))))+ (deps.findings/mark-entity-stale! :segment (:id object))+ (task.entity-check/trigger-entity-check-job!)))++(derive ::check-segment-dependents-on-delete :metabase/event)+(derive :event/segment-delete ::check-segment-dependents-on-delete)++(methodical/defmethod events/publish-event! ::check-segment-dependents-on-delete+ [_ {:keys [object]}]+ (when (premium-features/has-feature? :dependencies)+ (when (deps.findings/mark-immediate-dependents-stale! :segment (:id object))+ (task.entity-check/trigger-entity-check-job!))))(derive ::check-transform-dependents :metabase/event)(derive :event/transform-run-complete ::check-transform-dependents)@@ -328,7 +271,7 @@(methodical/defmethod events/publish-event! ::check-transform-dependents[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (when (deps.findings/mark-dependents-stale! :transform (:transform-id object))+ (when (deps.findings/mark-immediate-dependents-stale! :transform (:transform-id object))(task.entity-check/trigger-entity-check-job!))))(defn- synced-db->direct-dependents-of-changed-tables@@ -374,7 +317,7 @@(when (premium-features/has-feature? :dependencies)(let [changes (synced-db->direct-dependents-of-changed-tables db-id)](when (and (seq changes)- (deps.findings/mark-all-dependents-stale! {:table changes}))+ (deps.findings/mark-all-immediate-dependents-stale! {:table changes}))(task.entity-check/trigger-entity-check-job!)))));; ### Admin UI Table/Field Metadata Updates@@ -385,7 +328,8 @@(methodical/defmethod events/publish-event! ::check-table-metadata-update[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (deps.findings/mark-dependents-stale! :table (:id object))))+ (when (deps.findings/mark-immediate-dependents-stale! :table (:id object))+ (task.entity-check/trigger-entity-check-job!))))(derive ::check-field-metadata-update :metabase/event)(derive :event/field-update ::check-field-metadata-update)@@ -393,4 +337,5 @@(methodical/defmethod events/publish-event! ::check-field-metadata-update[_ {:keys [object]}](when (premium-features/has-feature? :dependencies)- (deps.findings/mark-dependents-stale! :table (:table_id object))))+ (when (deps.findings/mark-immediate-dependents-stale! :table (:table_id object))+ (task.entity-check/trigger-entity-check-job!))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.cljindex 116d79b2f10..c0282ae94e9 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj@@ -76,17 +76,31 @@(into [:enum] dependable-entities))(defn- mark-supported-dependents-stale!- "Given a map of {entity-type [entity-ids]}, mark all supported types as stale.+ "Given a map of {entity-type [entity-ids]}, mark all supported (analyzable) types as stale.+ For non-analyzable entities (e.g., tables), looks through to their dependents so the wave+ can reach analyzable entities beyond them.Returns true if any supported dependents were found."[dependents]- (let [supported-dependents (into {}- (filter (fn [[dep-type dep-ids]]- (and (analyzable-entities dep-type)- (seq dep-ids))))- dependents)]- (doseq [[dep-type dep-ids] supported-dependents]+ (let [{analyzable true non-analyzable false}+ (group-by (fn [[dep-type _]] (boolean (analyzable-entities dep-type)))+ (filter (fn [[_ dep-ids]] (seq dep-ids)) dependents))+ ;; For non-analyzable entities, find their direct dependents+ pass-through-dependents+ (when (seq non-analyzable)+ (let [key-seq (for [[dep-type dep-ids] non-analyzable+ id dep-ids]+ [dep-type id])+ deps-map (models.dependency/direct-dependents key-seq)]+ (models.dependency/group-nodes (into #{} cat (vals deps-map)))))+ ;; Merge the analyzable dependents with the pass-through dependents+ all-supported (merge-with into+ (into {} analyzable)+ (into {}+ (filter (fn [[dep-type _]] (analyzable-entities dep-type)))+ pass-through-dependents))]+ (doseq [[dep-type dep-ids] all-supported](deps.analysis-finding/mark-stale! dep-type dep-ids))- (boolean (seq supported-dependents))))+ (boolean (seq all-supported))))(mu/defn mark-dependents-stale! :- :boolean"Mark all transitive dependents of an entity as stale for re-analysis.@@ -100,15 +114,50 @@(let [dependents (models.dependency/transitive-dependents {entity-type [{:id entity-id}]})](mark-supported-dependents-stale! dependents)))-(defn mark-all-dependents-stale!- "Mark all transitive dependents of multiple entities as stale.+(defn mark-all-immediate-dependents-stale!+ "Mark the immediate dependents of multiple entities as stale.Takes a map of {entity-type entity-ids} where entity-ids is a set of IDs.+ Does NOT traverse transitively — the entity-check job loop propagates in waves.Returns true if any supported dependents were found."[type->entity-ids]- (let [type->objects (update-vals type->entity-ids #(map (fn [id] {:id id}) %))- dependents (models.dependency/transitive-dependents type->objects)]+ (let [key-seq (for [[entity-type ids] type->entity-ids+ id ids]+ [entity-type id])+ deps-map (models.dependency/direct-dependents key-seq)+ dependents (models.dependency/group-nodes+ (into #{} cat (vals deps-map)))](mark-supported-dependents-stale! dependents)))+(mu/defn mark-entity-stale!+ "Mark a single entity as stale for re-analysis by the background job."+ [entity-type :- AnalyzableEntityType+ entity-id :- pos-int?]+ (deps.analysis-finding/mark-stale! entity-type [entity-id]))++(mu/defn mark-immediate-dependents-stale! :- :boolean+ "Mark the immediate dependents of an entity as stale.+ Unlike [[mark-dependents-stale!]], this does NOT traverse transitively."+ [entity-type :- DependableEntityType+ entity-id :- pos-int?]+ (let [key-seq [[entity-type entity-id]]+ deps-map (models.dependency/direct-dependents key-seq)+ dependents (models.dependency/group-nodes+ (into #{} cat (vals deps-map)))]+ (mark-supported-dependents-stale! dependents)))++(defn- analyze-and-propagate!+ "Analyze an entity and mark its immediate dependents as stale.+ Wrapped in a transaction so that if marking dependents fails, the analysis+ is rolled back and the entity stays stale for retry on the next pass.+ The newly-stale dependents are picked up by the job's loop in+ `task.entity-check/check-entities!`, which drains all stale entities+ before returning — no need to re-trigger the job."+ [instance]+ (let [entity-type (deps.dependency-types/model->dependency-type (t2/model instance))]+ (t2/with-transaction [_conn]+ (upsert-analysis! instance)+ (mark-immediate-dependents-stale! entity-type (:id instance)))))+(mu/defn analyze-batch! :- nat-int?"Add or update analyses for a batch of entities.@@ -118,5 +167,9 @@batch-size :- pos-int?](let [instances (deps.analysis-finding/instances-for-analysis type batch-size)](lib-be/with-metadata-provider-cache- (analyze-instances! instances))+ (doseq [instance instances]+ (try (analyze-and-propagate! instance)+ (catch Exception e+ (log/errorf e "Analyzing entity %s %s failed"+ (t2/model instance) (:id instance))))))(count instances)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.cljnew file mode 100644index 00000000000..ade8ddfa90a--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj@@ -0,0 +1,131 @@+(ns metabase-enterprise.dependencies.models.dependency-status+ (:require+ [java-time.api :as t]+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase.app-db.core :as app-db]+ [metabase.models.interface :as mi]+ [methodical.core :as methodical]+ [toucan2.core :as t2]))++(methodical/defmethod t2/table-name :model/DependencyStatus [_model] :dependency_status)++(derive :model/DependencyStatus :metabase/model)++(t2/deftransforms :model/DependencyStatus+ {:entity_type mi/transform-keyword})++(defn mark-stale!+ "Mark entities of `entity-type` with ids in `entity-ids` as stale for dependency recalculation.+ Creates entries if they don't exist, or sets stale=true if they do.+ Resets retry state so previously-failed entities get a fresh chance.+ Uses [[app-db/update-or-insert!]] for cross-database atomicity."+ [entity-type entity-ids]+ (doseq [id entity-ids]+ (app-db/update-or-insert!+ :model/DependencyStatus+ {:entity_type entity-type :entity_id id}+ (fn [existing]+ (if existing+ {:stale true :fail_count 0 :next_retry_at nil :terminal false}+ {:stale true :dependency_analysis_version 0})))))++(defn upsert-status!+ "Upsert a dependency_status entry, setting stale=false, version to current,+ and clearing any failure state. Uses [[app-db/update-or-insert!]] for cross-database atomicity."+ [entity-type entity-id]+ (app-db/update-or-insert!+ :model/DependencyStatus+ {:entity_type entity-type :entity_id entity-id}+ (fn [_existing]+ {:dependency_analysis_version models.dependency/current-dependency-analysis-version+ :stale false+ :fail_count 0+ :next_retry_at nil})))++(defmulti hydrate-for-deps+ "Hydrate a batch of instances with data needed for dependency calculation.+ Dispatches on entity-type keyword. Default is identity (no hydration needed)."+ {:arglists '([entity-type instances])}+ (fn [entity-type _instances] entity-type))++(defmethod hydrate-for-deps :default [_ instances] instances)++(defmethod hydrate-for-deps :dashboard [_ instances]+ (t2/hydrate instances [:dashcards :series]))++(defn instances-for-dependency-calculation+ "Find a batch of instances of type `entity-type` and maximum size `batch-size` that need+ dependency calculation: stale=true OR version < current.+ Excludes terminal entities and entities whose retry delay hasn't elapsed.+ Returns full entity objects. Prioritizes stale over outdated.+ Uses Java time (not DB time) so tests with [[mt/with-clock]] work correctly."+ [entity-type batch-size]+ (let [model (deps.dependency-types/dependency-type->model entity-type)+ table-name (t2/table-name model)+ id-field (keyword (name table-name) "id")+ table-wildcard (keyword (name table-name) "*")+ now (t/offset-date-time)]+ (t2/select model+ {:select [table-wildcard]+ :from table-name+ :inner-join [:dependency_status [:and+ [:= :dependency_status.entity_id id-field]+ [:= :dependency_status.entity_type (name entity-type)]]]+ :where [:and+ ;; Needs processing: stale or version outdated+ [:or+ [:= :dependency_status.stale true]+ [:< :dependency_status.dependency_analysis_version+ models.dependency/current-dependency-analysis-version]]+ ;; Not terminally broken+ [:= :dependency_status.terminal false]+ ;; Retry delay has elapsed (or no delay set)+ [:or+ [:is :dependency_status.next_retry_at nil]+ [:<= :dependency_status.next_retry_at now]]]+ :order-by [[[:case [:= :dependency_status.stale true] [:inline 0] :else [:inline 1]]]]+ :limit batch-size})))++(defn has-pending-retries?+ "Returns true if there are any entities waiting to be retried (not terminal, with a set retry time)."+ []+ (t2/exists? :model/DependencyStatus+ :terminal false+ :next_retry_at [:not= nil]))++(defn has-stale-or-outdated?+ "Returns true if there are any entities needing dependency calculation:+ stale=true OR version < current, not terminal, and retry delay elapsed."+ []+ (let [now (t/offset-date-time)]+ (t2/exists? :model/DependencyStatus+ {:where [:and+ [:or+ [:= :stale true]+ [:< :dependency_analysis_version+ models.dependency/current-dependency-analysis-version]]+ [:= :terminal false]+ [:or+ [:is :next_retry_at nil]+ [:<= :next_retry_at now]]]})))++(defn record-failure!+ "Record a failed dependency calculation attempt for an entity.+ Increments fail_count and sets next_retry_at based on exponential backoff.+ If max retries exceeded, marks the entity as terminal."+ [entity-type entity-id max-retries delay-minutes]+ (when-let [status (t2/select-one :model/DependencyStatus+ :entity_type entity-type+ :entity_id entity-id)]+ (let [new-fail-count (inc (:fail_count status 0))]+ (if (> new-fail-count max-retries)+ (t2/update! :model/DependencyStatus (:id status)+ {:fail_count new-fail-count+ :terminal true+ :next_retry_at nil})+ (let [retry-minutes (* new-fail-count delay-minutes)]+ (t2/update! :model/DependencyStatus (:id status)+ {:fail_count new-fail-count+ :next_retry_at (when (pos? retry-minutes)+ (t/plus (t/offset-date-time) (t/minutes retry-minutes)))}))))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.cljindex 1463e08f3f8..425337090a8 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj@@ -3,157 +3,139 @@(see [[metabase-enterprise.dependencies.models.dependency]]) up to date, that is, makes surethe dependency table contains fresh entries.- This is done by finding entities whose dependency_analysis_version is less than- [[metabase-enterprise.dependencies.models.dependency/current-dependency-analysis-version]] and- updates this field. In most cases this triggers an event handler that calculates the dependencies- and populates the dependency table. For cards, the event is emitted by the job handler itself,- because the update of the record doesn't trigger the event handler. "+ This is done by querying the dependency_status table for entities that are stale or have an+ outdated dependency_analysis_version. The backfill task computes dependencies and updates+ the dependency_status table."(:require- [java-time.api :as t]+ [metabase-enterprise.dependencies.calculation :as deps.calculation][metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status][metabase-enterprise.dependencies.settings :as deps.settings][metabase-enterprise.dependencies.task-util :as deps.task-util]- [metabase.config.core :as config][metabase.events.core :as events][metabase.premium-features.core :as premium-features][metabase.task.core :as task]- [metabase.util :as u]+ [metabase.transforms.core :as transforms][metabase.util.log :as log][methodical.core :as methodical]- [toucan2.core :as t2])- (:import- (java.util Map Set)- (java.util.concurrent ConcurrentHashMap)))+ [toucan2.core :as t2]))(set! *warn-on-reflection* true)-(defn- current-millis- "Returns the current epoch millis. Uses java-time so that clock settings can be used in tests."- []- (t/to-millis-from-epoch (t/instant)))--(def ^:private entities- "The list of models to backfill.+(def ^:private entity-types+ "The list of entity types to backfill.- This is not the same as deps.dependency-types/models, because tables shouldn't be backfilled. Instead, links+ This is not the same as deps.dependency-types/dependency-types, because tables shouldn't be backfilled. Instead, linksinvolving tables are found via analysis of the other side of the relation."- [:model/Card- :model/Transform- :model/NativeQuerySnippet- :model/Dashboard- :model/Document- :model/Sandbox- :model/Segment- :model/Measure])--;; In-memory state for tracking failed entities-;; Stores {:model/Type {id {:fail-count N :next-retry-timestamp M}}}-(def ^:private retry-state- (zipmap entities (repeatedly #(ConcurrentHashMap.))))--;; Stores {:model/Type #{id1 id2 ...}} for entities that have exceeded MAX_RETRIES-(def ^:private terminally-broken- (zipmap entities (repeatedly #(ConcurrentHashMap/newKeySet))))+ [:card :transform :snippet :dashboard :document :sandbox :segment :measure])(def ^:private max-retries 5)-(defn- processable-ids [model-kw batch-size]- (let [target-version models.dependency/current-dependency-analysis-version- terminally-broken-ids ^Set (terminally-broken model-kw)- retry-state-map ^Map (retry-state model-kw)- current-time (current-millis)]- (into []- (comp- (map :id)- (filter (fn [id]- (and (not (.contains terminally-broken-ids id))- (let [entity-retry-info (.get retry-state-map id)]- (or (nil? entity-retry-info)- (>= current-time (:next-retry-timestamp entity-retry-info)))))))- (take batch-size))- (t2/reducible-select [model-kw :id] :dependency_analysis_version [:< target-version]))))--(def ^:private custom-backfill-events- {:model/Card :event/card-dependency-backfill- :model/Dashboard :event/dashboard-dependency-backfill})--(defn- custom-backfill-entity!- [model-kw event id target-version]- ;; We don't want to change the entity at all, we just want to update the dependency data and- ;; mark the entity as processed for this dependency analysis version.- (let [update-count (t2/update! model-kw id :dependency_analysis_version [:< target-version]- {:dependency_analysis_version target-version})]- (when-let [entity (and (pos? update-count)- (t2/select-one model-kw id))]- (events/publish-event! event {:object entity}))- update-count))--(defn- backfill-entity!- [model-kw id target-version]- (log/debug "Backfilling " (name model-kw) id)- (u/prog1- (t2/with-transaction [_]- (if-let [event (custom-backfill-events model-kw)]- (custom-backfill-entity! model-kw event id target-version)- (t2/update! model-kw id :dependency_analysis_version [:< target-version]- {:dependency_analysis_version target-version})))- (log/debug "Backfilled " (name model-kw) id)))+;;; ------------------------------ Post-deps cleanup multimethod ------------------------------++(defmulti post-deps-cleanup!+ "Perform entity-specific cleanup after dependencies have been replaced.+ For example, transforms need to clean up outdated downstream table->transform dependencies.+ Default is no-op."+ {:arglists '([entity-type entity])}+ (fn [entity-type _entity] entity-type))++(defmethod post-deps-cleanup! :default [_ _entity] nil)++(defmethod post-deps-cleanup! :transform [_ {:keys [id target] :as transform}]+ (let [db-id (transforms/transform-source-database transform)+ downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency+ :from_entity_type :table+ :to_entity_type :transform+ :to_entity_id id)+ downstream-tables (when (seq downstream-table-ids)+ (t2/select :model/Table :id [:in downstream-table-ids]))+ outdated-tables (remove (fn [table]+ (and (= (:schema table) (:schema target))+ (= (:name table) (:name target))+ (or (not db-id)+ (= db-id (:db_id table)))))+ downstream-tables)+ not-found-table-ids (remove (into #{} (map :id) downstream-tables)+ downstream-table-ids)]+ (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)+ (map :id) outdated-tables))]+ (t2/delete! :model/Dependency+ :from_entity_type :table+ :from_entity_id [:in outdated-downstream-table-ids]+ :to_entity_type :transform+ :to_entity_id id))))++;;; ------------------------------ Backfill orchestration ------------------------------++(defn- processable-instances [entity-type batch-size]+ (deps.dependency-status/hydrate-for-deps+ entity-type+ (deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))++(defn- compute-deps-for-entity!+ "Compute and store dependencies for an entity, then update its dependency_status.+ Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."+ [entity-type entity]+ (log/debug "Computing dependencies for" (name entity-type) (:id entity))+ (t2/with-transaction [_]+ (let [deps (deps.calculation/calculate-deps entity-type entity)]+ (models.dependency/replace-dependencies! entity-type (:id entity) deps)+ (post-deps-cleanup! entity-type entity)+ (deps.dependency-status/upsert-status! entity-type (:id entity)))))(defn- backfill-entity-batch!- [model-kw batch-size]- (let [model-name (name model-kw)- target-version models.dependency/current-dependency-analysis-version- retry-state-map ^Map (retry-state model-kw)- terminally-broken-set ^Set (terminally-broken model-kw)- ids (processable-ids model-kw batch-size)] ; Use the new get-processable-ids- (when (seq ids)- (log/infof "Processing a batch of %s %s(s)..." (count ids) model-name))- (reduce (fn [total id]+ [entity-type batch-size]+ (let [type-name (name entity-type)+ instances (processable-instances entity-type batch-size)]+ (when (seq instances)+ (log/infof "Processing a batch of %s %s(s)..." (count instances) type-name))+ (reduce (fn [total entity](+ total(try- ;; this should update the dependency table via a toucan2 update hook- (let [update-count (backfill-entity! model-kw id target-version)]- (.remove retry-state-map id)- update-count)+ (compute-deps-for-entity! entity-type entity)+ 1(catch Exception e- (let [current-time (current-millis)- entity-retry-info (.get retry-state-map id)- failure-count (inc (:fail-count entity-retry-info 0))- retry-minutes (* failure-count (deps.settings/dependency-backfill-delay-minutes))- new-next-retry-timestamp (+ current-time (* retry-minutes 60 1000))]- (if (> failure-count max-retries)- (do (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."- model-name id failure-count)- (.add terminally-broken-set id)- (.remove retry-state-map id)) ; Remove from retry map- (do (log/warnf e "Entity %s %s failed, failure count: %d, next retry no sooner than %d minutes."- model-name id failure-count retry-minutes)- (.put retry-state-map id {:fail-count failure-count- :next-retry-timestamp new-next-retry-timestamp}))))+ (let [id (:id entity)]+ (try+ (deps.dependency-status/record-failure!+ entity-type id max-retries+ (deps.settings/dependency-backfill-delay-minutes))+ (let [{:keys [fail_count terminal]} (t2/select-one :model/DependencyStatus+ :entity_type entity-type+ :entity_id id)]+ (if terminal+ (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."+ type-name id fail_count)+ (log/warnf e "Entity %s %s failed, failure count: %d."+ type-name id fail_count)))+ (catch Exception record-ex+ (log/errorf e "Entity %s %s failed during dependency calculation."+ type-name id)+ (log/errorf record-ex "Additionally, failed to record the failure for %s %s."+ type-name id))))0))))0- ids)))+ instances)))(defn- backfill-dependencies!"Job to backfill dependencies for all entities.Returns true if a full batch has been selected, nil or false otherwise."[](when (premium-features/has-feature? :dependencies)- (-> (reduce (fn [batch-size model-kw]+ (-> (reduce (fn [batch-size entity-type](if (< batch-size 1)(reduced 0)- (let [processed (backfill-entity-batch! model-kw batch-size)]+ (let [processed (backfill-entity-batch! entity-type batch-size)](when (pos? processed)(log/info "Updated" processed "entities."))(- batch-size processed))))(deps.settings/dependency-backfill-batch-size)- entities)+ entity-types)(< 1))))(defn- has-pending-retries? []- (some (fn [^Map model-retry-state]- (not (.isEmpty model-retry-state)))- (vals retry-state)))+ (deps.dependency-status/has-pending-retries?))(declare schedule-next-run!)@@ -186,12 +168,17 @@:delay-in-seconds delay-in-seconds:scheduler scheduler})))+(defn trigger-backfill-job!+ "Trigger the BackfillDependencies job to run after a brief delay.+ The 1-second delay ensures the calling transaction has committed before+ the job checks for stale entities."+ []+ (schedule-next-run! 1))+(defmethod task/init! ::DependencyBackfill [_](if (pos? (deps.settings/dependency-backfill-batch-size))- (schedule-next-run! (if config/is-test?- 0- (deps.task-util/job-initial-delay- (deps.settings/dependency-backfill-variance-minutes))))+ (schedule-next-run! (deps.task-util/job-initial-delay+ (deps.settings/dependency-backfill-variance-minutes)))(log/info "Not starting dependency backfill job because the batch size is not positive")))(derive ::backfill :metabase/event)@@ -201,4 +188,4 @@(methodical/defmethod events/publish-event! ::backfill[_ _](when (premium-features/has-feature? :dependencies)- (backfill-dependencies!)))+ (trigger-backfill-job!)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.cljindex 1f7648ddc61..c308ecda559 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj@@ -26,14 +26,15 @@(defn- check-entities!"Process entities, draining all stale entities before returning.- Returns true if the full batch was used (more work may remain)."+ Continues looping as long as there are stale entities — this is important because+ [[deps.findings/analyze-and-propagate!]] marks immediate dependents stale during processing,+ which need to be picked up in subsequent waves."[](when (premium-features/has-feature? :dependencies)(loop []- (let [used-full-batch? (process-one-batch!)]- (if (and used-full-batch? (deps.analysis-finding/has-stale-entities?))- (recur)- used-full-batch?)))))+ (process-one-batch!)+ (when (deps.analysis-finding/has-stale-entities?)+ (recur)))))(declare schedule-next-run!)@@ -70,7 +71,8 @@(log/info "Not starting dependency entity check job because the batch size is not positive")))(defn trigger-entity-check-job!- "Trigger the DependencyEntityCheck job to run immediately.- Use this when entities have been marked stale and need re-analysis."+ "Trigger the DependencyEntityCheck job to run after a brief delay.+ The 1-second delay ensures the calling transaction has committed before+ the job checks for stale entities."[]- (schedule-next-run! 0))+ (schedule-next-run! 1))diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.cljindex b8b4ebee670..56abd204bef 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj@@ -79,6 +79,7 @@"DataPermissions""DatabaseRouter""Dependency"+ "DependencyStatus""DocumentBookmark""HTTPAction""ImplicitAction"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.cljindex f466f68ce61..228a50695f1 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj@@ -7,7 +7,7 @@[metabase-enterprise.dependencies.core :as dependencies][metabase-enterprise.dependencies.events][metabase-enterprise.dependencies.findings :as dependencies.findings]- [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase.collections.models.collection :as collection][metabase.core.core :as mbc][metabase.events.core :as events]@@ -148,8 +148,9 @@(testing "POST /api/ee/dependencies/check-card"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [card (card/create-card! (basic-card) user)+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card"(assoc (card/create-card! (basic-card "Product question" :products)user)@@ -165,7 +166,7 @@(mt/with-temp [:model/User user {:email "[email protected]"}:model/Dashboard dashboard {}:model/Document document {}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [metadata-provider (mt/metadata-provider);; Create base card querying real orders tablebase-card (card/create-card! (basic-card) user)@@ -187,6 +188,7 @@:type :question:dataset_query proposed-query:result_metadata nil}]+ (deps.test/synchronously-run-backfill!)(is (=? {:success false:bad_cards [{:id (:id dashboard-card):dashboard_id (:id dashboard)@@ -202,7 +204,7 @@(mt/dataset test-data(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [mp (mt/metadata-provider);; Create base card querying real orders tablebase-card (card/create-card! (basic-card) user)@@ -221,6 +223,7 @@:type :question:dataset_query proposed-query:result_metadata nil}+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card" proposed-card)](is (=? {:success false:bad_cards [{:id (:id dependent-card)}]@@ -232,7 +235,7 @@(mt/dataset test-data(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [mp (mt/metadata-provider)orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))base-query (-> orders-query@@ -258,6 +261,7 @@:type :question:dataset_query proposed-query:result_metadata nil}+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card"proposed-card)](is (=? {:success false@@ -270,7 +274,7 @@(mt/dataset test-data(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [mp (mt/metadata-provider);; Create base card querying real orders tablebase-card (card/create-card! (basic-card) user)@@ -297,6 +301,7 @@:type :question:dataset_query proposed-query:result_metadata nil}+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card" proposed-card)](is (=? {:success false:bad_cards #{(:id dependent-card-1) (:id dependent-card-2)}@@ -308,7 +313,7 @@(mt/dataset test-data(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [mp (mt/metadata-provider);; Create base card querying real orders tablebase-card (card/create-card! (basic-card) user)@@ -325,6 +330,7 @@:type :question:dataset_query proposed-query:result_metadata nil}+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card" proposed-card)](is (=? {:success false:bad_cards [{:id (:id dependent-card)}]@@ -355,7 +361,7 @@(mt/with-temp [:model/User user {:email "[email protected]"}:model/NativeQuerySnippet {snippet-id :id snippet-name :name} {:name "filter-snippet":content "WHERE SUBTOTAL > 100"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [tag-name (str "snippet: " snippet-name)mp (mt/metadata-provider)native-query (-> (lib/native-query mp (format "SELECT * FROM ORDERS %s" (str "{{" tag-name "}}")))@@ -369,6 +375,7 @@:display :table:visualization_settings {}}user)+ _ (deps.test/synchronously-run-backfill!)proposed-content "WHERE NONEXISTENT_COLUMN > 100"response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-snippet"{:id snippet-id@@ -381,10 +388,11 @@(deftest graph-test(testing "GET /api/ee/dependencies/graph"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}](let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user){card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph" :id card-id-2 :type "card")creator {:email "[email protected]":id (:id user)}]@@ -441,10 +449,11 @@(testing "GET /api/ee/dependencies/graph with table as root node"(mt/dataset test-data(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}](let [_card-1 (card/create-card! (basic-card "Card 1" :orders) user)_card-2 (card/create-card! (basic-card "Card 2" :orders) user)+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph":id (mt/id :orders):type "table")]@@ -459,10 +468,11 @@(deftest dependents-test(testing "GET /api/ee/dependencies/graph/dependents"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}](let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user){card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id card-id-1:type "card"@@ -490,7 +500,7 @@(deftest ^:sequential dependents-multiple-types-test(testing "GET /api/ee/dependencies/graph/dependents with multiple dependent-types"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency :model/DashboardCard]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/DashboardCard](mt/with-temp [:model/User user {:email "[email protected]"}:model/Dashboard {dashboard-id :id} {:name "Test Dashboard"}](let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card "Base card - multi") user)@@ -501,7 +511,8 @@:col 0:size_x 4:size_y 4})- (while (#'dependencies.backfill/backfill-dependencies!))+ (events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(testing "single dependent-types value (backward compatibility)"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id card-id-1@@ -528,11 +539,12 @@(deftest ^:sequential dependents-multiple-card-types-test(testing "GET /api/ee/dependencies/graph/dependents with multiple dependent-card-types"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}](let [{dependency-card-id :id :as dependency-card} (card/create-card! (basic-card "Base card - cardtypes") user)_question-card (card/create-card! (assoc (wrap-card dependency-card) :name "Question card") user)_model-card (card/create-card! (assoc (wrap-card dependency-card) :name "Model card" :type :model) user)]+ (deps.test/synchronously-run-backfill!)(testing "single dependent-card-types value"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id dependency-card-id@@ -568,12 +580,13 @@(deftest graph-archived-card-test(testing "GET /api/ee/dependencies/graph with archived parameter"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}](let [base-card (card/create-card! (basic-card "Archived Base Card") user)dependent-card (card/create-card! (wrap-card base-card) user)](card/update-card! {:card-before-update base-card:card-updates {:archived true}})+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived card from dependencies"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph":id (:id dependent-card)@@ -594,12 +607,13 @@(deftest dependents-archived-card-test(testing "GET /api/ee/dependencies/graph/dependents with archived parameter"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}](let [base-card (card/create-card! (basic-card "Base Card") user)dependent-card (card/create-card! (wrap-card base-card) user)](card/update-card! {:card-before-update dependent-card:card-updates {:archived true}})+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived dependent"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents":id (:id base-card)@@ -620,7 +634,7 @@(testing "GET /api/ee/dependencies/graph/dependents?broken=true - only returns entities that are broken"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card dependent-card] (lib-be/with-metadata-provider-cache(let [model-card (create-model-card! user "Model Card - brokentest")@@ -630,7 +644,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card)(let [next-card (create-dependent-card-on-model! user model-card "Another Dependent Card - brokentest" {:table :products})]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id next-card))(run-analysis-for-card! (:id dependent-card))(let [response2 (mt/user-http-request :crowberto :get 200 (str "ee/dependencies/graph/dependents?broken=true&type=card&id=" (:id model-card)))]@@ -646,8 +660,9 @@(mt/with-non-admin-groups-no-root-collection-perms(mt/with-temp [:model/Collection collection {}:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [card (card/create-card! (assoc (basic-card) :collection_id (u/the-id collection)) user)]+ (deps.test/synchronously-run-backfill!)(testing "Returns 403 when user lacks read permissions"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :post 403 "ee/dependencies/check-card"@@ -701,11 +716,12 @@(mt/with-premium-features #{:dependencies :transforms-basic}(mt/with-temp [:model/User user {:email "[email protected]"}:model/Transform transform {}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [base-card (card/create-card! (basic-card) user)proposed-card {:id (:id base-card):type :question:dataset_query (:dataset_query base-card)}]+ (deps.test/synchronously-run-backfill!);; Mock errors-from-proposed-edits to report the transform as broken(with-redefs [dependencies/errors-from-proposed-edits(constantly {:transform {(:id transform) #{:some-error}}})]@@ -726,8 +742,9 @@(mt/with-non-admin-groups-no-root-collection-perms(mt/with-temp [:model/Collection collection {}:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [card (card/create-card! (assoc (basic-card) :collection_id (u/the-id collection)) user)]+ (deps.test/synchronously-run-backfill!)(testing "Returns 403 when user lacks read permissions"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :get 403 "ee/dependencies/graph"@@ -745,8 +762,9 @@(mt/with-non-admin-groups-no-root-collection-perms(mt/with-temp [:model/Collection collection {}:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [{card-id :id} (card/create-card! (assoc (basic-card) :collection_id (:id collection)) user)]+ (deps.test/synchronously-run-backfill!)(testing "Returns 403 when user lacks read permissions"(is (= "You don't have permissions to do that."(mt/user-http-request :rasta :get 403 "ee/dependencies/graph/dependents"@@ -770,7 +788,7 @@(mt/with-temp [:model/Collection readable-collection {}:model/Collection unreadable-collection {}:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [readable-base (card/create-card! (assoc (basic-card "Readable"):collection_id (:id readable-collection)) user)unreadable-base (card/create-card! (assoc (basic-card "Unreadable")@@ -778,6 +796,7 @@top-card (card/create-card! (assoc (wrap-two-cards readable-base unreadable-base):collection_id (:id readable-collection))user)]+ (deps.test/synchronously-run-backfill!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees complete upstream graph through readable path"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -810,7 +829,7 @@(mt/with-temp [:model/Collection readable-collection {}:model/Collection unreadable-collection {}:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [base-card (card/create-card! (assoc (basic-card) :collection_id (:id readable-collection)) user)readable-dependent (card/create-card! (assoc (wrap-card base-card):collection_id (:id readable-collection))@@ -818,6 +837,7 @@unreadable-dependent (card/create-card! (assoc (wrap-card base-card):collection_id (:id unreadable-collection))user)]+ (deps.test/synchronously-run-backfill!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only readable dependents"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"@@ -838,7 +858,7 @@(mt/with-temp [:model/Collection readable-collection {}:model/Collection unreadable-collection {}:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [base-card (card/create-card! (assoc (basic-card) :collection_id (:id readable-collection)) user)unreadable-middle (card/create-card! (assoc (wrap-card base-card):collection_id (:id unreadable-collection))@@ -849,6 +869,7 @@end-card (card/create-card! (assoc (wrap-two-cards unreadable-middle readable-alternate):collection_id (:id readable-collection))user)]+ (deps.test/synchronously-run-backfill!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "Diamond pattern: complete upstream graph via readable path"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -887,12 +908,13 @@(mt/with-temp [:model/Collection readable-collection {}:model/Collection unreadable-collection {}:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [unreadable-base (card/create-card! (assoc (basic-card "Unreadable"):collection_id (:id unreadable-collection)) user)top-card (card/create-card! (assoc (wrap-card unreadable-base):collection_id (:id readable-collection))user)]+ (deps.test/synchronously-run-backfill!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only the top card in the graph"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -918,7 +940,7 @@:model/User user {:email "[email protected]"}:model/NativeQuerySnippet {snippet-id :id snippet-name :name} {:name "test-snippet":content "WHERE ID > 10"}]- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [tag-name (str "snippet: " snippet-name)mp (mt/metadata-provider)native-query (fn []@@ -940,6 +962,7 @@:visualization_settings {}:collection_id (:id unreadable-collection)}user)]+ (deps.test/synchronously-run-backfill!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only readable cards as dependents of the snippet"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"@@ -961,12 +984,13 @@(mt/with-temp [:model/Collection readable-collection {}:model/User user {:email "[email protected]"}](mt/with-temp-copy-of-db- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-no-data-perms-for-all-users!(perms/set-table-permission! (perms/all-users-group) (mt/id :orders) :perms/view-data :blocked)(perms/set-table-permission! (perms/all-users-group) (mt/id :orders) :perms/create-queries :no)(let [card (card/create-card! (assoc (basic-card "Card on orders"):collection_id (:id readable-collection)) user)]+ (deps.test/synchronously-run-backfill!)(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)(testing "User sees only the card, table is filtered out"(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"@@ -986,13 +1010,14 @@(testing "Graph endpoints return dashboard data for cards in dashboards"(mt/with-premium-features #{:dependencies}(mt/with-current-user (mt/user->id :rasta)- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}:model/Dashboard dashboard {:name "Test Dashboard"}](let [base-card (card/create-card! (basic-card "Base Card") user)dashboard-card (card/create-card! (assoc (wrap-card base-card):dashboard_id (:id dashboard))user)]+ (deps.test/synchronously-run-backfill!)(testing "GET /api/ee/dependencies/graph returns dashboard with :id and :name"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id (:id dashboard-card)@@ -1018,13 +1043,14 @@(deftest graph-returns-document-for-cards-test(testing "Graph endpoints return document data for cards in documents"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user {:email "[email protected]"}:model/Document document {:name "Test Document"}](let [base-card (card/create-card! (basic-card "Base Card") user)document-card (card/create-card! (assoc (wrap-card base-card):document_id (:id document))user)]+ (deps.test/synchronously-run-backfill!)(testing "GET /api/ee/dependencies/graph returns document with :id and :name"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id (:id document-card)@@ -1057,6 +1083,7 @@:table_id products-id:definition {:filter [:> [:field price-field-id nil] 50]}}](events/publish-event! :event/segment-create {:object segment :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id segment-id:type "segment")@@ -1071,7 +1098,7 @@(deftest graph-segment-dependents-test(testing "GET /api/ee/dependencies/graph/dependents with segment"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [products-id (mt/id :products)price-field-id (mt/id :products :price)](mt/with-temp [:model/Segment {segment-id :id :as segment} {:name "High Value Products"@@ -1085,6 +1112,7 @@query-with-segment (-> (lib/query mp products)(lib/filter (lib.metadata/segment mp segment-id)))card (card/create-card! (card-with-query "Card using segment" query-with-segment) user)+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents":id segment-id:type "segment"@@ -1107,6 +1135,7 @@:definition (-> (lib/query mp products)(lib/aggregate (lib/sum price)))}](events/publish-event! :event/measure-create {:object measure :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id measure-id:type "measure")@@ -1132,6 +1161,7 @@:definition (-> (lib/query mp products)(lib/aggregate (lib/sum price)))}](events/publish-event! :event/measure-create {:object measure-a :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [mp' (mt/metadata-provider)](mt/with-temp [:model/Measure {measure-b-id :id :as measure-b} {:name "Measure B":table_id products-id@@ -1139,6 +1169,7 @@(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id)(lib/sum rating))))}](events/publish-event! :event/measure-create {:object measure-b :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id measure-b-id:type "measure")@@ -1158,7 +1189,7 @@(deftest graph-measure-dependents-test(testing "GET /api/ee/dependencies/graph/dependents with measure"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [mp (mt/metadata-provider)products-id (mt/id :products)products (lib.metadata/table mp products-id)@@ -1174,6 +1205,7 @@query-with-measure (-> (lib/query mp' products)(lib/aggregate (lib.metadata/measure mp' measure-id)))card (card/create-card! (card-with-query "Card using measure" query-with-measure) user)+ _ (deps.test/synchronously-run-backfill!)response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents":id measure-id:type "measure"@@ -1234,7 +1266,7 @@;; Archive measure B (the middle of the chain)(mt/user-http-request :crowberto :put 200 (str "measure/" measure-b-id){:archived true :revision_message "Archive middle measure"})- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "after archiving measure B, it and measure A are excluded (chain broken)"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph":id measure-c-id@@ -1304,7 +1336,9 @@:dataset_query (->> referenced-card-id(lib.metadata/card mp)(lib/query mp))}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id referenced-card-id) :user-id (mt/user->id :crowberto)})+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id unreffed-card-id) :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=unreftest")](is (=? {:data [{:id unreffed-card-id:type "card"@@ -1317,10 +1351,11 @@(let [mp (mt/metadata-provider)](mt/with-temp [:model/Table {unreffed-table-id :id} {:name "Unreferenced Table - unreftest"}:model/Table {referenced-table-id :id} {:name "Referenced Table - unreftest"}- :model/Card _card {:name "Referencing Card"- :type :question- :dataset_query (lib/query mp (lib.metadata/table mp referenced-table-id))}]- (while (#'dependencies.backfill/backfill-dependencies!))+ :model/Card {card-id :id} {:name "Referencing Card"+ :type :question+ :dataset_query (lib/query mp (lib.metadata/table mp referenced-table-id))}]+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=unreftest")](is (=? {:data [{:id unreffed-table-id:type "table"@@ -1350,7 +1385,7 @@:output-schema "PUBLIC":output-table "referenced_transform_table":transform-id referenced-transform-id}})- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(try(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")](is (=? {:data [{:id unreffed-transform-id@@ -1376,10 +1411,11 @@:type :snippet:snippet-name snippet-name:snippet-id referenced-snippet-id}}))]- (mt/with-temp [:model/Card _card {:name "Card using snippet"- :type :question- :dataset_query native-query}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (mt/with-temp [:model/Card {card-id :id} {:name "Card using snippet"+ :type :question+ :dataset_query native-query}]+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=unreftest")](is (=? {:data [{:id unreffed-snippet-id:type "snippet"@@ -1391,15 +1427,15 @@(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - unreftest"}:model/Dashboard {referenced-dashboard-id :id} {:name "Referenced Dashboard - unreftest"}- :model/Document _ {:name "Document with dashboard link"- :dependency_analysis_version 0- :document {:type "doc"- :content [{:type "paragraph"- :content [{:type "smartLink"- :attrs {:entityId referenced-dashboard-id- :model "dashboard"}}]}]}- :content_type "application/json+vnd.prose-mirror"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ :model/Document {document-id :id} {:name "Document with dashboard link"+ :document {:type "doc"+ :content [{:type "paragraph"+ :content [{:type "smartLink"+ :attrs {:entityId referenced-dashboard-id+ :model "dashboard"}}]}]}+ :content_type "application/json+vnd.prose-mirror"}]+ (events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id document-id) :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=unreftest")](is (=? {:data [{:id unreffed-dashboard-id:type "dashboard"@@ -1417,7 +1453,8 @@:attrs {:entityId referenced-document-id:model "document"}}]}]}:content_type "application/json+vnd.prose-mirror"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id unreffed-document-id) :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=unreftest")](is (=? {:data [{:id unreffed-document-id:type "document"@@ -1436,7 +1473,7 @@:model/Sandbox {sandbox-id :id} {:group_id group-id:table_id (mt/id :products):card_id sandbox-card-id}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=sandbox")](is (=? {:data [{:id sandbox-id:type "sandbox"@@ -1454,7 +1491,7 @@:model/Card {unreffed-metric-id :id} {:name "B - Unreferenced Metric - cardtype":type :metric:dataset_query (lib/query mp products)}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "filtering by model only"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=model&query=cardtype")](is (=? {:data [{:id unreffed-model-id@@ -1493,7 +1530,7 @@:type :question:archived true:dataset_query (lib/query mp products)}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived card"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=archivedtest")card-ids (set (map :id (:data response)))]@@ -1511,7 +1548,7 @@(mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - archivedtest"}:model/Dashboard {archived-dashboard-id :id} {:name "Archived Unreferenced Dashboard - archivedtest":archived true}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived dashboard"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest")dashboard-ids (set (map :id (:data response)))]@@ -1529,7 +1566,7 @@(mt/with-temp [:model/Document {unreffed-document-id :id} {:name "Unreferenced Document - archivedtest"}:model/Document {archived-document-id :id} {:name "Archived Unreferenced Document - archivedtest":archived true}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived document"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest")document-ids (set (map :id (:data response)))]@@ -1549,7 +1586,7 @@:model/NativeQuerySnippet {archived-snippet-id :id} {:name "Archived Unreferenced Snippet - archivedtest":content "WHERE ID > 20":archived true}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived snippet"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest")snippet-ids (set (map :id (:data response)))]@@ -1575,7 +1612,7 @@:archived true}](events/publish-event! :event/segment-create {:object unreffed-segment :user-id (mt/user->id :crowberto)})(events/publish-event! :event/segment-create {:object archived-segment :user-id (mt/user->id :crowberto)})- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived segment"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest")segment-ids (set (map :id (:data response)))]@@ -1600,7 +1637,7 @@:db_id (mt/id):active true:visibility_type "hidden"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes inactive and hidden tables"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest")table-ids (set (map :id (:data response)))]@@ -1620,7 +1657,7 @@(mt/with-temp [:model/Table {table1-id :id} {:name "Table 1 - unreftest"}:model/Table {table2-id :id} {:name "Table 2 - unreftest"}:model/Table {table3-id :id} {:name "Table 3 - unreftest"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(is (=? {:data [{:id table1-id} {:id table2-id}]:total 3:offset 0@@ -1669,7 +1706,7 @@:archived true}](events/publish-event! :event/measure-create {:object unreffed-measure :user-id (mt/user->id :crowberto)})(events/publish-event! :event/measure-create {:object archived-measure :user-id (mt/user->id :crowberto)})- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "archived=false (default) excludes archived measure"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest")measure-ids (set (map :id (:data response)))]@@ -1703,7 +1740,7 @@:model/Card {card-regular :id} {:name "Card Regular - personalcolltest":type :question:dataset_query (lib/query mp products)}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "include-personal-collections=false (default) excludes cards in personal collections"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=personalcolltest")card-ids (set (map :id (:data response)))]@@ -1727,7 +1764,7 @@:model/Dashboard {dash-in-personal :id} {:name "Dashboard in Personal - personalcolltest":collection_id personal-coll-id}:model/Dashboard {dash-regular :id} {:name "Dashboard Regular - personalcolltest"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(testing "include-personal-collections=false (default) excludes dashboards in personal collections"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=personalcolltest")dashboard-ids (set (map :id (:data response)))]@@ -1751,7 +1788,7 @@:model/Document _ {:name "F Document sorttest"}:model/Segment _ {:name "G Segment sorttest"}:model/Measure _ {:name "H Measure sorttest"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(doseq [sort-direction [:asc :desc]](let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced"@@ -1816,7 +1853,7 @@:table_id table1-id}:model/Measure _ {:name "Measure with Table 2 sorttest":table_id table2-id}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(doseq [sort-direction [:asc :desc]](let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced"@@ -1846,7 +1883,7 @@:collection_id collection1-id}:model/NativeQuerySnippet _ {:name "SQL snippets sorttest"}:model/Transform _ {:name "Transforms sorttest"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(doseq [sort-direction [:asc :desc]](let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced"@@ -1865,7 +1902,7 @@(testing "GET /api/ee/dependencies/graph/breaking - returns entities that are SOURCE of downstream errors"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card dependent-card](lib-be/with-metadata-provider-cache@@ -1876,7 +1913,7 @@(break-model-card! model-card));; Run analysis in a fresh metadata provider cache session to detect the broken reference(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card)))(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=brokentest")](is (= [(:id model-card)] (mapv :id (:data response)))@@ -1886,7 +1923,7 @@(testing "GET /api/ee/dependencies/graph/breaking - types parameter filters results"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card-1 model-card-2 dependent-card-1 dependent-card-2](lib-be/with-metadata-provider-cache@@ -1901,7 +1938,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(testing "filtering by card returns only card sources"@@ -1918,7 +1955,7 @@(testing "GET /api/ee/dependencies/graph/breaking with archived parameter for source cards"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[active-model archived-model dependent-card-1 dependent-card-2](lib-be/with-metadata-provider-cache@@ -1936,7 +1973,7 @@(break-model-card! (t2/select-one :model/Card :id (:id archived-model))));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(testing "archived=false (default) excludes archived source card"@@ -1954,7 +1991,7 @@(testing "GET /api/ee/dependencies/graph/breaking - model breaking multiple dependents appears once"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card dependent-card-1 dependent-card-2](lib-be/with-metadata-provider-cache@@ -1967,7 +2004,7 @@(break-model-card! model-card));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=multipledependents")@@ -1982,7 +2019,7 @@:model/User creator {:email "[email protected]"}:model/Collection {personal-coll-id :id} {:personal_owner_id user-id:name "Test Personal Collection"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-in-personal model-regular dependent-card-1 dependent-card-2](lib-be/with-metadata-provider-cache@@ -1998,7 +2035,7 @@(break-model-card! model-regular));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(testing "include-personal-collections=false (default) excludes source cards in personal collections"@@ -2016,7 +2053,7 @@(testing "GET /api/ee/dependencies/graph/breaking - should paginate results"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card-1 model-card-2 dependent-card-1 dependent-card-2](lib-be/with-metadata-provider-cache@@ -2031,7 +2068,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(is (=? {:data [{:id (:id model-card-1)}]@@ -2049,7 +2086,7 @@(testing "GET /api/ee/dependencies/graph/breaking - sorting by name"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card-a model-card-b dependent-card-a dependent-card-b](lib-be/with-metadata-provider-cache@@ -2064,7 +2101,7 @@(break-model-card! model-card-b));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-a))(run-analysis-for-card! (:id dependent-card-b)))(doseq [sort-direction [:asc :desc]]@@ -2086,7 +2123,7 @@(mt/with-temp [:model/User user {:email "[email protected]"}:model/Collection {collection1-id :id} {:name "B Collection"}:model/Collection {collection2-id :id} {:name "A Collection"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-in-coll1 model-in-coll2 dependent-card-1 dependent-card-2](lib-be/with-metadata-provider-cache@@ -2101,7 +2138,7 @@(break-model-card! model-in-coll2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(doseq [sort-direction [:asc :desc]]@@ -2121,7 +2158,7 @@(testing "GET /api/ee/dependencies/graph/breaking - sorting by dependents with errors count"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card-1 model-card-2 dependent-card-1a dependent-card-1b dependent-card-2a](lib-be/with-metadata-provider-cache@@ -2138,7 +2175,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1a))(run-analysis-for-card! (:id dependent-card-1b))(run-analysis-for-card! (:id dependent-card-2a)))@@ -2159,7 +2196,7 @@(testing "GET /api/ee/dependencies/graph/breaking - sorting by dependents with errors count"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session;; Model 1 breaks 2 dependent cards (2 unique analyzed entities with errors);; Model 2 breaks 1 dependent card (1 unique analyzed entity with errors)@@ -2177,7 +2214,7 @@(break-model-card! model-card-2));; Run analysis in a fresh metadata provider cache session to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1a))(run-analysis-for-card! (:id dependent-card-1b))(run-analysis-for-card! (:id dependent-card-2a)))@@ -2210,7 +2247,7 @@(defmacro ^:private with-dependents-test![[user-binding base-card-binding] & body]`(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user# {:email "[email protected]"}](let [~user-binding user#~base-card-binding (card/create-card! (basic-card "Base") user#)]@@ -2222,6 +2259,7 @@(create-dependent! base-card user "Alpha")(create-dependent! base-card user "Beta")(create-dependent! base-card user "Gamma")+ (deps.test/synchronously-run-backfill!)(testing "filters by name"(is (=? [{:data {:name "Alpha"}}](get-dependents base-card-id :query "Alpha"))))@@ -2237,6 +2275,7 @@(mt/with-temp [:model/Collection {coll-id :id} {:name "SpecialCollection"}](create-dependent! base-card user "Card in root")(create-dependent! base-card user "Card in collection" :collection_id coll-id)+ (deps.test/synchronously-run-backfill!)(is (=? [{:data {:name "Card in collection"}}](get-dependents base-card-id :query "SpecialCollection")))))))@@ -2249,6 +2288,7 @@(create-dependent! base-card user "In Personal" :collection_id personal-coll-id)(create-dependent! base-card user "In Sub" :collection_id sub-coll-id)(create-dependent! base-card user "Regular")+ (deps.test/synchronously-run-backfill!)(testing "default excludes personal collections"(is (=? [{:data {:name "Regular"}}](get-dependents base-card-id))))@@ -2261,6 +2301,7 @@(create-dependent! base-card user "C Card")(create-dependent! base-card user "A Card")(create-dependent! base-card user "B Card")+ (deps.test/synchronously-run-backfill!)(is (=? [{:data {:name "A Card"}} {:data {:name "B Card"}} {:data {:name "C Card"}}](get-dependents base-card-id :sort-column :name :sort-direction :asc)))(is (=? [{:data {:name "C Card"}} {:data {:name "B Card"}} {:data {:name "A Card"}}]@@ -2275,6 +2316,7 @@(create-dependent! base-card user "In C" :collection_id coll-c)(create-dependent! base-card user "In A" :collection_id coll-a)(create-dependent! base-card user "In B" :collection_id coll-b)+ (deps.test/synchronously-run-backfill!)(is (=? [{:data {:name "In A"}} {:data {:name "In B"}} {:data {:name "In C"}}](get-dependents base-card-id :sort-column :location :sort-direction :asc)))(is (=? [{:data {:name "In C"}} {:data {:name "In B"}} {:data {:name "In A"}}]@@ -2286,6 +2328,7 @@(let [{low-id :id} (create-dependent! base-card user "Low"){mid-id :id} (create-dependent! base-card user "Mid"){high-id :id} (create-dependent! base-card user "High")]+ (deps.test/synchronously-run-backfill!)(t2/update! :model/Card low-id {:view_count 10})(t2/update! :model/Card mid-id {:view_count 50})(t2/update! :model/Card high-id {:view_count 100})@@ -2303,7 +2346,8 @@{high-id :id} (create-dependent! base-card user "High")](t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id base-card-id:row 0 :col 0 :size_x 4 :size_y 4})- (while (#'dependencies.backfill/backfill-dependencies!))+ (events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(t2/update! :model/Card low-id {:view_count 10})(t2/update! :model/Card high-id {:view_count 100})(is (=? [{:data {:view_count 10}} {:data {:view_count 100}} {:data {:view_count 200}}]@@ -2318,6 +2362,7 @@(create-dependent! base-card user "A Match")(create-dependent! base-card user "B Match")(create-dependent! base-card user "Should not appear")+ (deps.test/synchronously-run-backfill!)(is (=? [{:data {:name "A Match"}} {:data {:name "B Match"}} {:data {:name "C Match"}}](get-dependents base-card-id :query "Match" :sort-column :name :sort-direction :asc))))))@@ -2386,7 +2431,7 @@(testing "GET /api/ee/dependencies/graph/breaking - pagination and sorting work with error visibility filtering"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create cards in one metadata provider cache session(let [[model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a archived-dep-b](lib-be/with-metadata-provider-cache@@ -2408,7 +2453,7 @@(break-model-card! model-card-b));; Run analysis to detect broken references(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id visible-dep-a))(run-analysis-for-card! (:id visible-dep-b))(run-analysis-for-card! (:id archived-dep-a))@@ -2482,7 +2527,7 @@(testing "GET /api/ee/dependencies/graph/breaking - sorting counts only visible errors, not archived"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Setup: Model A has fewer VISIBLE errors but more TOTAL errors (due to archived dependents);; Model B has more VISIBLE errors but fewer TOTAL errors;; If sorting counts all errors: A (3 total) > B (2 total) -> B first in asc@@ -2547,11 +2592,11 @@(deftest ^:sequential unreferenced-pagination-with-archived-items-test(testing "GET /api/ee/dependencies/graph/unreferenced - pagination works correctly with archived items"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Dependency]+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus](mt/with-temp [:model/Card _ {:name "A Card - unreftest" :archived true}:model/Card {card2-id :id} {:name "B Card - unreftest"}:model/Card {card3-id :id} {:name "C Card - unreftest"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "card"@@ -2569,18 +2614,20 @@(deftest ^:sequential unreferenced-pagination-with-archived-dependents-test(testing "GET /api/ee/dependencies/graph/unreferenced - should return items if all dependents are archived"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Dependency]+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus](mt/with-temp [:model/Card {card1-id :id, :as card1} {:name "A Card - unreftest"}:model/Card card2 {:name "B Card - unreftest"}- :model/Card _ {:name "C Card - unreftest"+ :model/Card {card3-id :id} {:name "C Card - unreftest":dataset_query (wrap-card-query card1):archived true}:model/Card {card4-id :id} {:name "D Card - unreftest":dataset_query (wrap-card-query card2)}- :model/Card _ {:name "E Card - unreftest"+ :model/Card {card5-id :id} {:name "E Card - unreftest":dataset_query (wrap-card-query card2):archived true}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (doseq [id [card1-id (:id card2) card3-id card4-id card5-id]]+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id id) :user-id (mt/user->id :crowberto)}))+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "card"@@ -2621,6 +2668,7 @@:type :query:query {:source-table table-1-id}}}](events/publish-event! :event/card-create {:object card :user-id (:creator_id card)})+ (deps.test/synchronously-run-backfill!)(testing "after creating card that depends on table-1, only table-2 is unused"(is (= #{table-2-id}(->> (mt/user-http-request :crowberto :get 200 "table" :unused-only true)@@ -2635,7 +2683,7 @@(testing "GET /api/ee/dependencies/graph/broken - returns broken dependents filtered by source"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Setup: two models, each with one dependent(let [[model-card-1 model-card-2 dependent-card-1 dependent-card-2](lib-be/with-metadata-provider-cache@@ -2650,7 +2698,7 @@(break-model-card! model-card-1)(break-model-card! model-card-2))(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-1))(run-analysis-for-card! (:id dependent-card-2)))(testing "returns only broken dependents for the specified source (not other sources)"@@ -2664,7 +2712,7 @@(testing "GET /api/ee/dependencies/graph/broken - count matches and sorting works"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Setup: one model with two dependents (named for sorting)(let [[model-card dependent-card-a dependent-card-b](lib-be/with-metadata-provider-cache@@ -2677,7 +2725,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card-a))(run-analysis-for-card! (:id dependent-card-b)))(testing "count matches dependents-with-errors from /graph/breaking"@@ -2721,7 +2769,7 @@(mt/with-non-admin-groups-no-root-collection-perms(mt/with-temp [:model/User user {:email "[email protected]"}:model/Collection {coll-id :id} {:name "Private Collection"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError](let [[model-card dependent-card](lib-be/with-metadata-provider-cache(let [model-card (create-model-card! user "Model Card - permtest" :collection-id coll-id)@@ -2731,7 +2779,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card)));; Admin can access(is (sequential? (mt/user-http-request :crowberto :get 200@@ -2745,7 +2793,7 @@(testing "GET /api/ee/dependencies/graph/broken - does not report transitive breakage"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; Create chain: model-1 -> model-2 -> card;; When model-1 breaks:;; - model-2 gets error with source_entity_id=model-1@@ -2766,7 +2814,7 @@(break-model-card! model-card-1));; Run analysis for both model-2 and dependent-card(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id model-card-2))(run-analysis-for-card! (:id dependent-card)));; Query /graph/broken for model-1 - should only return model-2, not dependent-card@@ -2781,7 +2829,7 @@(testing "GET /api/ee/dependencies/graph/broken - filters by dependent-types"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError](let [[model-card dependent-card](lib-be/with-metadata-provider-cache(let [model-card (create-model-card! user "Model Card - deptypestest")@@ -2790,7 +2838,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-card)))(testing "filtering by card type returns the dependent card"(let [response (mt/user-http-request :crowberto :get 200@@ -2811,7 +2859,7 @@:model/User creator {:email "[email protected]"}:model/Collection {personal-coll-id :id} {:personal_owner_id user-id:name "Test Personal Collection"}]- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError];; one model, two dependents: one in personal collection, one in regular collection(let [[model-card dependent-in-personal dependent-regular](lib-be/with-metadata-provider-cache@@ -2825,7 +2873,7 @@(lib-be/with-metadata-provider-cache(break-model-card! model-card))(lib-be/with-metadata-provider-cache- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(run-analysis-for-card! (:id dependent-in-personal))(run-analysis-for-card! (:id dependent-regular)))(testing "include-personal-collections=false (default) excludes broken dependents in personal collections"@@ -2845,12 +2893,12 @@(deftest ^:sequential unreferenced-table-owner-test(testing "GET /api/ee/dependencies/unreferenced - table owner is returned"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Dependency]+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus](mt/with-temp [:model/Table {table1-id :id} {:name "User Owned Table - ownertest":owner_user_id (mt/user->id :crowberto)}:model/Table {table2-id :id} {:name "Email Owned Table - ownertest":owner_email "[email protected]"}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "table" :query "ownertest"@@ -2871,7 +2919,7 @@(mt/with-premium-features #{:dependencies}(let [mp (mt/metadata-provider)products (lib.metadata/table mp (mt/id :products))]- (mt/with-model-cleanup [:model/Dependency]+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus](mt/with-temp [:model/Transform {transform1-id :id} {:name "User Owned Transform - ownertest":owner_user_id (mt/user->id :crowberto):source {:type :query@@ -2884,7 +2932,7 @@:query (lib/query mp products)}:target {:schema "PUBLIC":name "email_owned_transform_table"}}]- (while (#'dependencies.backfill/backfill-dependencies!))+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200"ee/dependencies/graph/unreferenced":types "transform" :query "ownertest"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.cljindex 8435aba1e23..bc182d4d8bb 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj@@ -22,12 +22,12 @@:measure #{}:segment #{}:table #{products-id}}- (calculation/upstream-deps:card prod-card)))+ (calculation/calculate-deps :card prod-card)))(is (= {:card #{prod-card-id}:measure #{}:segment #{}:table #{}}- (calculation/upstream-deps:card widget-card))))))+ (calculation/calculate-deps :card widget-card))))))(deftest ^:parallel upstream-deps-card-join-test(let [mp (mt/metadata-provider)@@ -42,12 +42,12 @@:measure #{}:segment #{}:table #{products-id}}- (calculation/upstream-deps:card products-card)))+ (calculation/calculate-deps :card products-card)))(is (= {:card #{products-card-id}:measure #{}:segment #{}:table #{orders-id}}- (calculation/upstream-deps:card joined-card))))))+ (calculation/calculate-deps :card joined-card))))))(deftest ^:parallel upstream-deps-card-implicit-join-fields-test(let [mp (mt/metadata-provider)@@ -64,7 +64,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id users-id}}- (calculation/upstream-deps:card card))))))+ (calculation/calculate-deps :card card))))))(deftest ^:parallel upstream-deps-card-implicit-join-filter-test(let [mp (mt/metadata-provider)@@ -79,7 +79,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id}}- (calculation/upstream-deps:card card))))))+ (calculation/calculate-deps :card card))))))(deftest ^:parallel upstream-deps-card-implicit-join-breakout-test(let [mp (mt/metadata-provider)@@ -94,7 +94,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id}}- (calculation/upstream-deps:card card))))))+ (calculation/calculate-deps :card card))))))(deftest ^:parallel upstream-deps-card-implicit-join-aggregation-test(let [mp (mt/metadata-provider)@@ -109,7 +109,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id}}- (calculation/upstream-deps:card card))))))+ (calculation/calculate-deps :card card))))))(deftest ^:parallel upstream-deps-card-implicit-join-order-by-test(let [mp (mt/metadata-provider)@@ -124,7 +124,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id}}- (calculation/upstream-deps:card card))))))+ (calculation/calculate-deps :card card))))))(deftest ^:parallel upstream-deps-transform-implicit-join-fields-test(mt/with-premium-features #{:transforms-basic}@@ -147,7 +147,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id users-id}}- (calculation/upstream-deps:transform transform)))))))+ (calculation/calculate-deps :transform transform)))))))(deftest ^:parallel upstream-deps-transform-implicit-join-breakout-test(mt/with-premium-features #{:transforms-basic}@@ -170,7 +170,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id}}- (calculation/upstream-deps:transform transform)))))))+ (calculation/calculate-deps :transform transform)))))))(deftest ^:parallel upstream-deps-transform-implicit-join-aggregation-test(mt/with-premium-features #{:transforms-basic}@@ -192,7 +192,7 @@:measure #{}:segment #{}:table #{checkins-id venues-id}}- (calculation/upstream-deps:transform transform)))))))+ (calculation/calculate-deps :transform transform)))))))(deftest ^:parallel upstream-deps-python-transform-test(mt/with-premium-features #{:transforms-basic}@@ -207,7 +207,7 @@:target {:schema "PUBLIC":name "test_output"}}](is (= {:table #{products-id orders-id}}- (calculation/upstream-deps:transform transform)))))))+ (calculation/calculate-deps :transform transform)))))))(deftest ^:parallel upstream-deps-native-transform-with-table-tag-test(testing "GHY-3258: native transform with a table template tag should include the table in dependencies"@@ -224,7 +224,7 @@:query query}:target {:schema "PUBLIC":name "test_output"}}]- (is (=? {:table #(contains? % products-id)} (calculation/upstream-deps:transform transform))))))))+ (is (=? {:table #(contains? % products-id)} (calculation/calculate-deps :transform transform))))))))(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test(let [mp (mt/metadata-provider)@@ -255,10 +255,10 @@:measure #{}:segment #{}:table #{products-id}}- (calculation/upstream-deps:card category-values-card)))+ (calculation/calculate-deps :card category-values-card)))(is (= {:card #{category-values-card-id}:table #{products-id}}- (calculation/upstream-deps:card native-card))))))+ (calculation/calculate-deps :card native-card))))))(deftest ^:parallel upstream-deps-dashboard-test(let [mp (mt/metadata-provider)@@ -272,7 +272,7 @@(t2/hydrate :dashcards))](is (= {:card #{card-id}:dashboard #{}}- (calculation/upstream-deps:dashboard dashboard)))))))+ (calculation/calculate-deps :dashboard dashboard)))))))(deftest ^:parallel upstream-deps-dashboard-with-series-test(let [mp (mt/metadata-provider)@@ -288,14 +288,10 @@:model/DashboardCardSeries _ {:dashboardcard_id dashcard-id:card_id series-card-id:position 0}]- (let [dashcards (t2/select :model/DashboardCard :dashboard_id dashboard-id)- series-card-ids (when (seq dashcards)- (t2/select-fn-set :card_id :model/DashboardCardSeries- :dashboardcard_id [:in (map :id dashcards)]))- dashboard (assoc dashboard :dashcards dashcards :series-card-ids series-card-ids)]+ (let [dashboard (t2/hydrate dashboard [:dashcards :series])](is (= {:card #{main-card-id series-card-id}:dashboard #{}}- (calculation/upstream-deps:dashboard dashboard)))))))+ (calculation/calculate-deps :dashboard dashboard)))))))(deftest ^:parallel upstream-deps-dashboard-with-parameter-source-test(let [mp (mt/metadata-provider)@@ -317,7 +313,7 @@(assoc :dashcards dashcards))](is (= {:card #{filtered-card-id category-values-card-id}:dashboard #{}}- (calculation/upstream-deps:dashboard dashboard)))))))+ (calculation/calculate-deps :dashboard dashboard)))))))(deftest ^:parallel upstream-deps-dashboard-with-click-behavior-test(let [mp (mt/metadata-provider)@@ -338,7 +334,7 @@:dashcards dashcards}](is (= {:card #{count-card-id target-card-id}:dashboard #{}}- (calculation/upstream-deps:dashboard dashboard)))))))+ (calculation/calculate-deps :dashboard dashboard)))))))(deftest ^:parallel upstream-deps-dashboard-with-column-click-behavior-test(let [mp (mt/metadata-provider)@@ -361,7 +357,7 @@:dashcards dashcards}](is (= {:card #{table-card-id target-card-id}:dashboard #{}}- (calculation/upstream-deps:dashboard dashboard)))))))+ (calculation/calculate-deps :dashboard dashboard)))))))(deftest ^:parallel upstream-deps-dashboard-with-dashboard-click-behavior-test(let [mp (mt/metadata-provider)@@ -382,7 +378,7 @@:dashcards dashcards}](is (= {:card #{count-card-id}:dashboard #{target-dashboard-id}}- (calculation/upstream-deps:dashboard dashboard)))))))+ (calculation/calculate-deps :dashboard dashboard)))))))(deftest ^:parallel upstream-deps-dashboard-with-column-dashboard-click-behavior-test(let [mp (mt/metadata-provider)@@ -405,7 +401,7 @@:dashcards dashcards}](is (= {:card #{table-card-id}:dashboard #{target-dashboard-id}}- (calculation/upstream-deps:dashboard dashboard)))))))+ (calculation/calculate-deps :dashboard dashboard)))))))(deftest ^:parallel upstream-deps-document-test(let [mp (mt/metadata-provider)@@ -431,7 +427,7 @@(is (= {:card #{card-id embedded-card-id}:dashboard #{dashboard-id}:table #{products-id}}- (calculation/upstream-deps:document document))))))+ (calculation/calculate-deps :document document))))))(deftest upstream-deps-sandbox-test(mt/with-premium-features #{:sandboxes}@@ -441,7 +437,7 @@:table_id (mt/id :products):card_id sandbox-card-id}](is (= {:card #{sandbox-card-id}}- (calculation/upstream-deps:sandbox sandbox))))))+ (calculation/calculate-deps :sandbox sandbox))))))(deftest ^:parallel upstream-deps-segment-test(let [products-id (mt/id :products)@@ -451,7 +447,7 @@(mt/with-temp [:model/Segment segment {:table_id products-id:definition {:filter [:> [:field price-field-id nil] 50]}}](is (= {:segment #{} :table #{products-id}}- (calculation/upstream-deps:segment segment)))))+ (calculation/calculate-deps :segment segment)))))(testing "segment depending on another segment"(mt/with-temp [:model/Segment {segment-a-id :id :as segment-a} {:table_id products-id@@ -462,11 +458,11 @@[:= [:field category-field-id nil] "Widget"]]}}](testing "base segment depends only on table"(is (= {:segment #{} :table #{products-id}}- (calculation/upstream-deps:segment segment-a))))+ (calculation/calculate-deps :segment segment-a))))(testing "dependent segment depends on both table and segment"(is (= {:table #{products-id}:segment #{segment-a-id}}- (calculation/upstream-deps:segment segment-b))))))))+ (calculation/calculate-deps :segment segment-b))))))))(deftest ^:parallel upstream-deps-card-with-multiple-segments-test(testing "Card using multiple segments depends on all of them"@@ -487,7 +483,7 @@:measure #{}:segment #{segment-a-id segment-b-id}:table #{products-id}}- (calculation/upstream-deps:card card)))))))+ (calculation/calculate-deps :card card)))))))(deftest ^:parallel upstream-deps-segment-with-multiple-segments-test(testing "Segment depending on multiple other segments tracks all dependencies"@@ -506,7 +502,7 @@[:> [:field rating-field-id nil] 4]]}}](is (= {:table #{products-id}:segment #{segment-a-id segment-b-id}}- (calculation/upstream-deps:segment segment-c)))))))+ (calculation/calculate-deps :segment segment-c)))))))(deftest ^:parallel upstream-deps-segment-implicit-join-test(testing "Segment depending on implicitly joined field adds dep on that field's table"@@ -518,7 +514,7 @@:definition {:filter [:= [:field venue-name-field-id {:source-field venue-fk-field-id}] "Bird's Nest"]}}](is (= {:segment #{}:table #{checkins-id venues-id}}- (calculation/upstream-deps:segment segment)))))))+ (calculation/calculate-deps :segment segment)))))))(deftest upstream-deps-measure-test(let [mp (mt/metadata-provider)@@ -531,7 +527,7 @@:definition (-> (lib/query mp orders)(lib/aggregate (lib/sum quantity)))}](is (= {:measure #{} :segment #{} :table #{orders-id}}- (calculation/upstream-deps:measure measure)))))+ (calculation/calculate-deps :measure measure)))))(testing "measure depending on another measure"(mt/with-temp [:model/Measure {measure-a-id :id :as measure-a} {:name "Measure A"@@ -545,10 +541,10 @@(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id) 100)))}](testing "base measure depends only on table"(is (= {:measure #{} :segment #{} :table #{orders-id}}- (calculation/upstream-deps:measure measure-a))))+ (calculation/calculate-deps :measure measure-a))))(testing "dependent measure depends on both table and measure"(is (= {:measure #{measure-a-id} :segment #{} :table #{orders-id}}- (calculation/upstream-deps:measure measure-b))))))))))+ (calculation/calculate-deps :measure measure-b))))))))))(deftest upstream-deps-measure-with-multiple-measures-test(testing "Measure depending on multiple other measures tracks all dependencies"@@ -572,7 +568,7 @@(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id)(lib.metadata/measure mp' measure-b-id))))}](is (= {:measure #{measure-a-id measure-b-id} :segment #{} :table #{orders-id}}- (calculation/upstream-deps:measure measure-c)))))))))+ (calculation/calculate-deps :measure measure-c)))))))))(deftest upstream-deps-measure-implicit-join-test(testing "Measure depending on implicitly joined field adds dep on that field's table"@@ -587,7 +583,7 @@:table_id checkins-id:definition (lib/aggregate base-query (lib/sum venue-price))}](is (= {:measure #{} :segment #{} :table #{checkins-id venues-id}}- (calculation/upstream-deps:measure measure)))))))+ (calculation/calculate-deps :measure measure)))))))(deftest upstream-deps-card-with-measure-test(testing "Card using a measure depends on that measure"@@ -603,7 +599,7 @@(mt/with-temp [:model/Card card {:dataset_query (-> (lib/query mp' orders)(lib/aggregate (lib.metadata/measure mp' measure-id)))}](is (= {:card #{} :measure #{measure-id} :segment #{} :table #{orders-id}}- (calculation/upstream-deps:card card)))))))))+ (calculation/calculate-deps :card card)))))))))(deftest upstream-deps-measure-with-segment-test(testing "Measure with conditional aggregation using segment depends on that segment"@@ -621,4 +617,4 @@:definition (-> (lib/query mp' orders)(lib/aggregate (lib/sum-where quantity (lib/ref segment-meta))))}](is (= {:measure #{} :segment #{segment-id} :table #{orders-id}}- (calculation/upstream-deps:measure measure)))))))))+ (calculation/calculate-deps :measure measure)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex 897d692b86b..27fc7016d87 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -1,20 +1,18 @@(ns metabase-enterprise.dependencies.events-test(:require- [clojure.string :as str][clojure.test :refer [deftest is testing]]- [metabase-enterprise.dependencies.async :as async]- [metabase-enterprise.dependencies.calculation :as deps.calculation][metabase-enterprise.dependencies.events][metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]- [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase.api.common :as api][metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.test :as mt]- [metabase.util.log.capture :as log.capture][toucan2.core :as t2]))(comment@@ -23,11 +21,17 @@(defn- run-with-dependencies-setup! [thunk](mt/with-test-user :rasta(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/AnalysisFinding]+ (mt/with-model-cleanup [:model/AnalysisFinding :model/DependencyStatus](lib-be/with-metadata-provider-cache(let [mp (mt/metadata-provider)](thunk mp)))))))+(defn- assert-stale+ "Assert that the given entity is marked stale in dependency_status."+ [entity-type entity-id]+ (is (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true)+ (str "Expected " (name entity-type) " " entity-id " to be marked stale")))+(deftest dashboard-update-sets-correct-dependencies(run-with-dependencies-setup!(fn [mp]@@ -51,6 +55,8 @@:card_id series-card-id:position 0}](events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})+ (assert-stale :dashboard dashboard-id)+ (deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :dashboard:from_entity_id dashboard-id:to_entity_type :card@@ -107,6 +113,7 @@:targetId column-click-target-dashboard-id}}}}}](let [updated-dashboard (t2/select-one :model/Dashboard :id dashboard-id)](events/publish-event! :event/dashboard-update {:object updated-dashboard :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :dashboard:from_entity_id dashboard-id:to_entity_type :card@@ -163,6 +170,8 @@{:type "cardEmbed":attrs {:id embedded-card-id}}]}}](events/publish-event! :event/document-create {:object document :user-id api/*current-user-id*})+ (assert-stale :document document-id)+ (deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :document:from_entity_id document-id:to_entity_type :card@@ -183,6 +192,7 @@:model "table"}}]}]})](t2/update! :model/Document document-id updated-doc)(events/publish-event! :event/document-update {:object updated-doc :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :document:from_entity_id document-id:to_entity_type :dashboard@@ -208,6 +218,8 @@:table_id (mt/id :products):card_id card1-id}](events/publish-event! :event/sandbox-create {:object sandbox :user-id api/*current-user-id*})+ (assert-stale :sandbox sandbox-id)+ (deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :sandbox:from_entity_id sandbox-id:to_entity_type :card@@ -217,6 +229,7 @@(t2/update! :model/Sandbox sandbox-id (assoc sandbox :card_id card2-id))(let [updated-sandbox (t2/select-one :model/Sandbox :id sandbox-id)](events/publish-event! :event/sandbox-update {:object updated-sandbox :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :sandbox:from_entity_id sandbox-id:to_entity_type :card@@ -227,64 +240,44 @@(events/publish-event! :event/sandbox-delete {:object sandbox :user-id api/*current-user-id*})(is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))))))-(deftest card-dependency-calculation-error-handling-test- (testing "When dependency calculation throws an error, it should be logged and the version should still be updated"+(deftest card-marks-stale-on-create-update-test+ (testing "Card create/update events mark the entity stale in dependency_status"(run-with-dependencies-setup!(fn [mp]- (let [products-id (mt/id :products)- products (lib.metadata/table mp products-id)]+ (let [products (lib.metadata/table mp (mt/id :products))](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:card (fn [_]- (throw (ex-info "Dependency calculation failed"- {:card-id card-id})))]- (testing "on create event"- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})- (is (some #(and (= "Dependency calculation failed" (ex-message (:e %)))- (= :error (:level %))- (str/includes? (:message %) "{:entity-type :card")- (str/includes? (:message %) (str card-id)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Card :id card-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))- (testing "on update event"- (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})- (is (some #(and (= "Dependency calculation failed" (ex-message (:e %)))- (= :error (:level %))- (str/includes? (:message %) "{:entity-type :card")- (str/includes? (:message %) (str card-id)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Card :id card-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))))))))))--(deftest snippet-dependency-calculation-error-handling-test- (testing "When snippet dependency calculation throws an error, it should be logged and the version should still be updated"+ (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})+ (assert-stale :card card-id)+ (deps.test/synchronously-run-backfill!)+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :products)))+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})+ (assert-stale :card card-id)))))))++(deftest snippet-marks-stale-on-create-update-test+ (testing "Snippet create/update events mark the entity stale in dependency_status"(run-with-dependencies-setup!(fn [_](mt/with-temp [:model/NativeQuerySnippet {snippet-id :id :as snippet} {:name "test snippet":content "SELECT 1"}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:snippet (fn [_]- (throw (ex-info "Snippet dependency calculation failed"- {:snippet-id snippet-id})))]- (testing "on create event"- (events/publish-event! :event/snippet-create {:object snippet :user-id api/*current-user-id*})- (is (some #(and (= "Snippet dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet :id snippet-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id snippet-id :from_entity_type :snippet))))- (testing "on update event"- (events/publish-event! :event/snippet-update {:object snippet :user-id api/*current-user-id*})- (is (some #(and (= "Snippet dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet :id snippet-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id snippet-id :from_entity_type :snippet)))))))))))+ (events/publish-event! :event/snippet-create {:object snippet :user-id api/*current-user-id*})+ (assert-stale :snippet snippet-id)+ (events/publish-event! :event/snippet-update {:object snippet :user-id api/*current-user-id*})+ (assert-stale :snippet snippet-id))))))++(deftest mark-stale-failure-does-not-propagate-test+ (testing "When mark-stale! throws, the event handler catches the error and the API call is not affected"+ (run-with-dependencies-setup!+ (fn [mp]+ (let [products (lib.metadata/table mp (mt/id :products))]+ (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]+ (with-redefs [deps.dependency-status/mark-stale!+ (fn [_ _] (throw (ex-info "Simulated DB failure" {})))]+ ;; Should not throw — the error is caught and logged+ (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*}))+ ;; Entity should NOT be stale (mark-stale! failed)+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))+ "Entity should not be marked stale when mark-stale! fails")))))))(deftest ^:sequential native-transform-updates-dependencies-test(testing "native transform update events trigger dependency calculations"@@ -294,8 +287,8 @@:type :query}](mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}](events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (assert-stale :transform transform-id)+ (deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,:to_entity_type :table,@@ -310,8 +303,8 @@:type :query}](mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}](events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (assert-stale :transform transform-id)+ (deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,:to_entity_type :table,@@ -333,8 +326,8 @@"import pandas as pd\n\ndef transform(orders):\n return orders"}](mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}](events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))+ (assert-stale :transform transform-id)+ (deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,:to_entity_type :table,@@ -355,8 +348,6 @@:output-table :test_table:transform-id transform-id}:user-id api/*current-user-id*})- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,@@ -398,17 +389,21 @@(events/publish-event! :event/update-transform{:object transform:user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,:to_entity_id transform-id}](t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "changing target update"- (events/publish-event! :event/update-transform- {:object (assoc-in transform [:target :name] "test_table2")- :user-id api/*current-user-id*})- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})+ (let [updated (t2/select-one :model/Transform :id transform-id)]+ (events/publish-event! :event/update-transform+ {:object updated+ :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test(testing "query transform update events handles downstream dependencies"@@ -437,127 +432,21 @@(events/publish-event! :event/update-transform{:object transform:user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,:to_entity_id transform-id}](t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "changing target update"- (events/publish-event! :event/update-transform- {:object (assoc-in transform [:target :name] "test_table2")- :user-id api/*current-user-id*})- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))--(deftest transform-dependency-calculation-error-handling-test- (testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup!- (fn [_]- (mt/with-temp [:model/Transform {transform-id :id :as transform} {}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:transform (fn [_]- (throw (ex-info "Transform dependency calculation failed"- {:transform-id transform-id})))]- (testing "on create event"- (events/publish-event! :event/create-transform {:object transform :user-id api/*current-user-id*})- (is (some #(and (= "Transform dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform))))- (testing "on update event"- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (is (some #(and (= "Transform dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))))--(deftest dashboard-dependency-calculation-error-handling-test- (testing "When dashboard dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup!- (fn [_]- (mt/with-temp [:model/Dashboard {dashboard-id :id :as dashboard} {}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:dashboard (fn [_]- (throw (ex-info "Dashboard dependency calculation failed"- {:dashboard-id dashboard-id})))]- (testing "on create event"- (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})- (is (some #(and (= "Dashboard dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Dashboard :id dashboard-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard))))- (testing "on update event"- (events/publish-event! :event/dashboard-update {:object dashboard :user-id api/*current-user-id*})- (is (some #(and (= "Dashboard dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Dashboard :id dashboard-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))))))))--(deftest document-dependency-calculation-error-handling-test- (testing "When document dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup!- (fn [_]- (mt/with-temp [:model/Document {document-id :id :as document} {:content_type "application/json+vnd.prose-mirror"- :document {:type "doc" :content []}}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:document (fn [_]- (throw (ex-info "Document dependency calculation failed"- {:document-id document-id})))]- (testing "on create event"- (events/publish-event! :event/document-create {:object document :user-id api/*current-user-id*})- (is (some #(and (= "Document dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Document :id document-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document))))- (testing "on update event"- (events/publish-event! :event/document-update {:object document :user-id api/*current-user-id*})- (is (some #(and (= "Document dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Document :id document-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document)))))))))))--(deftest sandbox-dependency-calculation-error-handling-test- (testing "When sandbox dependency calculation throws an error, it should be logged and the version should still be updated"- (mt/with-premium-features #{:sandboxes}- (run-with-dependencies-setup!- (fn [_]- (mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "test group"}- :model/Card {card-id :id} {}- :model/Sandbox {sandbox-id :id :as sandbox} {:group_id group-id- :table_id (mt/id :products)- :card_id card-id}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:sandbox (fn [_]- (throw (ex-info "Sandbox dependency calculation failed"- {:sandbox-id sandbox-id})))]- (testing "on create event"- (events/publish-event! :event/sandbox-create {:object sandbox :user-id api/*current-user-id*})- (is (some #(and (= "Sandbox dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Sandbox :id sandbox-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))- (testing "on update event"- (events/publish-event! :event/sandbox-update {:object sandbox :user-id api/*current-user-id*})- (is (some #(and (= "Sandbox dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Sandbox :id sandbox-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))))))+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})+ (let [updated (t2/select-one :model/Transform :id transform-id)]+ (events/publish-event! :event/update-transform+ {:object updated+ :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))(deftest segment-update-sets-correct-dependencies(run-with-dependencies-setup!@@ -569,6 +458,8 @@:definition {:filter [:> [:field price-field-id nil] 50]}}](testing "creating a segment creates dependency to its table"(events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})+ (assert-stale :segment segment-id)+ (deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :segment:from_entity_id segment-id:to_entity_type :table@@ -579,6 +470,7 @@(t2/update! :model/Segment segment-id {:definition {:filter [:= [:field category-field-id nil] "Widget"]}})(let [updated-segment (t2/select-one :model/Segment :id segment-id)](events/publish-event! :event/segment-update {:object updated-segment :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :segment:from_entity_id segment-id:to_entity_type :table@@ -599,11 +491,13 @@:definition {:filter [:> [:field price-field-id nil] 50]}}](testing "creating a card using a segment creates dependencies to both segment and table"(events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query {:database (mt/id):type :query:query {:source-table products-id:filter [:segment segment-id]}}}](events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :card:from_entity_id card-id:to_entity_type :segment@@ -615,35 +509,6 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))))))))))-(deftest segment-dependency-calculation-error-handling-test- (testing "When segment dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup!- (fn [_]- (let [products-id (mt/id :products)- price-field-id (mt/id :products :price)]- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id- :definition {:filter [:> [:field price-field-id nil] 50]}}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:segment (fn [_]- (throw (ex-info "Segment dependency calculation failed"- {:segment-id segment-id})))]- (testing "on create event"- (events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})- (is (some #(and (= "Segment dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Segment :id segment-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))- (testing "on update event"- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})- (is (some #(and (= "Segment dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Segment :id segment-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))))))))-(deftest measure-update-sets-correct-dependencies(run-with-dependencies-setup!(fn [mp]@@ -657,6 +522,8 @@(lib/aggregate (lib/sum quantity)))}](testing "creating a measure creates dependency to its table"(events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})+ (assert-stale :measure measure-id)+ (deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :measure:from_entity_id measure-id:to_entity_type :table@@ -668,6 +535,7 @@(lib/aggregate (lib/sum subtotal)))})(let [updated-measure (t2/select-one :model/Measure :id measure-id)](events/publish-event! :event/measure-update {:object updated-measure :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :measure:from_entity_id measure-id:to_entity_type :table@@ -697,6 +565,7 @@(testing "creating a measure that references another measure creates dependencies to both"(events/publish-event! :event/measure-create {:object measure1 :user-id api/*current-user-id*})(events/publish-event! :event/measure-create {:object measure2 :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :measure:from_entity_id measure2-id:to_entity_type :table@@ -718,12 +587,14 @@:definition (-> (lib/query mp orders)(lib/aggregate (lib/sum quantity)))}](events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(testing "creating a card using a measure creates dependencies to both measure and table"(let [mp' (mt/metadata-provider)query (-> (lib/query mp' orders)(lib/aggregate (lib.metadata/measure mp' measure-id)))](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query query}](events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :card:from_entity_id card-id:to_entity_type :measure@@ -735,36 +606,7 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card)))))))))))))-(deftest measure-dependency-calculation-error-handling-test- (testing "When measure dependency calculation throws an error, it should be logged and the version should still be updated"- (run-with-dependencies-setup!- (fn [mp]- (let [orders-id (mt/id :orders)- orders (lib.metadata/table mp orders-id)- quantity (lib.metadata/field mp (mt/id :orders :quantity))]- (mt/with-temp [:model/Measure {measure-id :id :as measure} {:table_id orders-id- :definition (-> (lib/query mp orders)- (lib/aggregate (lib/sum quantity)))}]- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]- (with-redefs [deps.calculation/upstream-deps:measure (fn [_]- (throw (ex-info "Measure dependency calculation failed"- {:measure-id measure-id})))]- (testing "on create event"- (events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})- (is (some #(and (= "Measure dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Measure :id measure-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))- (testing "on update event"- (events/publish-event! :event/measure-update {:object measure :user-id api/*current-user-id*})- (is (some #(and (= "Measure dependency calculation failed" (ex-message (:e %)))- (= :error (:level %)))- (messages)))- (is (= models.dependency/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Measure :id measure-id)))- (is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))))+;; === Analysis finding tests (these are unchanged — they use AnalysisFinding, not dependency_analysis_version) ===(defn- assert-has-analyses"Asserts that a given set of entities have the appropriate analysis version.@@ -788,7 +630,7 @@(deftest ^:sequential card-update-works-with-no-analyses-test(run-with-dependencies-setup!(fn [mp]- (testing "Card updates should analyze the card and mark dependents stale for async processing"+ (testing "Card updates should mark the card stale and trigger the background job"(let [products-id (mt/id :products)products (lib.metadata/table mp products-id)](mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}@@ -797,10 +639,10 @@:from_entity_id child-card-id:to_entity_type :card:to_entity_id parent-card-id}]- (with-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ ;; No analysis record should exist yet - mark-entity-stale! is a no-op for never-analyzed entities(assert-has-analyses- {:card {parent-card-id -1}})+ {:card {parent-card-id nil}})(is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFinding:analyzed_entity_type :card:analyzed_entity_id child-card-id))@@ -809,13 +651,12 @@(deftest ^:sequential card-update-updates-analyses-test(run-with-dependencies-setup!(fn [mp]- (testing "Card updates should update the card's analysis and mark dependents stale"+ (testing "Card updates should mark the card stale, then the job re-analyzes and propagates"(let [products-id (mt/id :products)orders-id (mt/id :orders)products (lib.metadata/table mp products-id)orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]+ old-version models.analysis-finding/*current-analysis-finding-version*](mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}:model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}@@ -831,83 +672,41 @@{:card {parent-card-id old-versionchild-card-id old-versionother-card-id old-version}}))- (t2/with-transaction [_conn]- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (with-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})))- (testing "Parent should be re-analyzed synchronously"+ (testing "card update only marks direct dependents stale - not its own analysis"+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ (assert-has-analyses+ {:card {parent-card-id old-version+ child-card-id -1+ other-card-id old-version}}))+ (testing "If the version changes, the job picks up only the stale cards"+ (binding [models.analysis-finding/*current-analysis-finding-version* (inc old-version)](assert-has-analyses- {:card {parent-card-id new-version}}))- (testing "Child should be marked stale (not re-analyzed yet)"- (let [child-finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id child-card-id)]- (is (= old-version (:analysis_version child-finding)))- (is (true? (:stale child-finding)))))- (testing "Other card should be unchanged"- (let [other-finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id other-card-id)]- (is (= old-version (:analysis_version other-finding)))- (is (false? (:stale other-finding))))))))))))+ {:card {parent-card-id old-version+ child-card-id -1+ other-card-id old-version}})))))))))-(deftest ^:sequential stale-entities-are-processed-by-job-test- (run-with-dependencies-setup!- (fn [mp]- (testing "Stale entities are re-analyzed when the job runs"- (let [products (lib.metadata/table mp (mt/id :products))- current-version models.analysis-finding/*current-analysis-finding-version*]- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}- :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-card-id- :to_entity_type :card- :to_entity_id parent-card-id}]- ;; Create initial analyses for both cards- (deps.findings/upsert-analysis! parent-card)- (deps.findings/upsert-analysis! child-card)- ;; Update parent - this marks child as stale- (t2/with-transaction [_conn]- (with-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))- ;; Verify child is stale- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id child-card-id))- "Child should be marked stale after parent update"))- ;; Run the job to process stale entities- (deps.findings/analyze-batch! :card 10)- ;; Verify child is no longer stale and has been re-analyzed- (let [child-finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id child-card-id)]- (is (false? (:stale child-finding))- "Child should no longer be stale after job runs")- (is (= current-version (:analysis_version child-finding))- "Child should have current analysis version"))))))))+;;; ------------------------------------------------ Analysis propagation tests ------------------------------------------------(deftest ^:sequential card-update-transaction-rollback-test(run-with-dependencies-setup!(fn [mp]- (testing "If marking dependents stale fails, analysis upsert is rolled back"+ (testing "If marking immediate dependents stale fails, analysis upsert is rolled back"(let [products (lib.metadata/table mp (mt/id :products))old-version models.analysis-finding/*current-analysis-finding-version*new-version (inc old-version)](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]- ;; Create initial analysis(deps.findings/upsert-analysis! card)(testing "Initial analysis exists"(is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding:analyzed_entity_type :card:analyzed_entity_id card-id))))- ;; Try to update with mark-dependents-stale! throwing an exception(binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (with-redefs [async/submit! (fn [f] (f))- deps.findings/mark-dependents-stale! (fn [_ _] (throw (ex-info "Simulated failure" {})))]+ (with-redefs [deps.findings/mark-immediate-dependents-stale!+ (fn [_ _] (throw (ex-info "Simulated failure" {})))]+ ;; analyze-and-propagate! wraps in a transaction, so the upsert should be rolled back(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated failure"- (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})))))- ;; Verify analysis was NOT updated (rolled back)- (testing "Analysis should be unchanged after failed event"+ (#'deps.findings/analyze-and-propagate! card)))))+ (testing "Analysis should be unchanged after rolled-back transaction"(is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding:analyzed_entity_type :card:analyzed_entity_id card-id))))))))))@@ -915,48 +714,37 @@(deftest ^:sequential card-update-triggers-native-cards-test(run-with-dependencies-setup!(fn [mp]- (let [old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc old-version)]- (mt/with-temp [:model/Card- {parent-id :id :as parent}- {:dataset_query (lib/native-query mp "select * from products")}-- :model/Card- {child-id :id :as child}- {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}-- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}]+ (testing "Card update marks entity stale, and the entity-check job re-analyzes it and marks dependents stale"+ (mt/with-temp [:model/Card {parent-id :id :as parent} {:dataset_query (lib/native-query mp "select * from products")}+ :model/Card {child-id :id :as child} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}+ :model/Dependency _ {:from_entity_type :card :from_entity_id child-id+ :to_entity_type :card :to_entity_id parent-id}](deps.findings/upsert-analysis! parent)(deps.findings/upsert-analysis! child)- (t2/with-transaction [_conn]- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (with-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update {:object parent- :previous-object parent- :user-id api/*current-user-id*})- (testing "Native card update should trigger analysis"- (assert-has-analyses- {:card {parent-id new-version}}))- (testing "Its dependents should be marked stale"- (let [child-finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id child-id)]- (is (= old-version (:analysis_version child-finding)))- (is (true? (:stale child-finding)))))))))))))+ ;; Event marks entity stale in analysis_finding+ (events/publish-event! :event/card-update {:object parent :previous-object parent :user-id api/*current-user-id*})+ (testing "Parent card should be marked stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id parent-id))))+ ;; Run entity-check job to process stale entities+ (#'task.entity-check/check-entities!)+ (testing "Parent should be re-analyzed"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id parent-id))))+ (testing "Child should have been re-analyzed via wave propagation"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id child-id)))))))))(deftest ^:sequential card-update-stops-on-transforms-test(run-with-dependencies-setup!(fn [mp]- (testing "Card updates should continue analyzing through a transform"- (let [products-id (mt/id :products)- orders-id (mt/id :orders)- products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]+ (testing "Card update marks immediate dependents stale including transforms, wave propagates through"+ (let [products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))+ old-version models.analysis-finding/*current-analysis-finding-version*](mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}:model/Transform {transform-id :id :as transform} {:source {:type :query:query (lib/query mp products)}@@ -965,137 +753,91 @@:name "sample":type :table}}:model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-card-id- :to_entity_type :transform- :to_entity_id transform-id}- :model/Dependency _ {:from_entity_type :transform- :from_entity_id transform-id- :to_entity_type :card- :to_entity_id parent-card-id}]+ :model/Dependency _ {:from_entity_type :transform :from_entity_id transform-id+ :to_entity_type :card :to_entity_id parent-card-id}+ :model/Dependency _ {:from_entity_type :card :from_entity_id child-card-id+ :to_entity_type :transform :to_entity_id transform-id}](deps.findings/upsert-analysis! parent-card)(deps.findings/upsert-analysis! transform)(deps.findings/upsert-analysis! child-card)- (testing "Checking that initial analyses exist"- (assert-has-analyses- {:card {parent-card-id old-version- child-card-id old-version}- :transform {transform-id old-version}}))- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (t2/with-transaction [_conn]- (with-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})- (testing "parent card should be re-analyzed synchronously"- (is (=? {:analysis_version new-version :stale false}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id parent-card-id))))- (testing "transform (direct dependent) should be marked stale"- (is (=? {:analysis_version old-version :stale true}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :transform- :analyzed_entity_id transform-id))))- (testing "child card (transitive dependent) should be marked stale"- (is (=? {:analysis_version old-version :stale true}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id child-card-id)))))))))))))+ (assert-has-analyses+ {:card {parent-card-id old-version child-card-id old-version}+ :transform {transform-id old-version}})+ ;; Event marks parent stale in analysis_finding, which triggers entity-check+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ ;; Run entity-check job — should propagate through transform to child via waves+ (#'task.entity-check/check-entities!)+ (testing "All entities should be re-analyzed after wave propagation"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id parent-card-id)))+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :transform :analyzed_entity_id transform-id)))+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id child-card-id))))))))))(deftest ^:sequential transform-update-works-with-no-analyses-test(run-with-dependencies-setup!(fn [mp]- (testing "transform update creates new analyses"- (let [products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id :as transform} {:source {:type :query- :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :transform- :to_entity_id transform-id}]- (events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})- (assert-has-analyses- ;; child cards shouldn't be updated on transform update, only on run- {:card {card-id nil}- :transform {transform-id -1}})))))))+ (testing "Transform update does not error when no analyses exist yet"+ (let [products (lib.metadata/table mp (mt/id :products))]+ (mt/with-temp [:model/Transform transform {:source {:type :query+ :query (lib/query mp products)}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ ;; Should not throw even when no analysis findings exist+ (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})))))))))(deftest ^:sequential transform-update-updates-analyses-test(run-with-dependencies-setup!(fn [mp]- (testing "transform update updates analyses when they exist"- (let [products-id (mt/id :products)- orders-id (mt/id :orders)- products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id :as transform} {:source {:type :query- :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :transform- :to_entity_id transform-id}]- (deps.findings/upsert-analysis! card)- (deps.findings/upsert-analysis! transform)- (deps.findings/upsert-analysis! other-card)- (testing "Checking that initial analyses exist"- (assert-has-analyses- {:card {card-id old-version- other-card-id old-version}- :transform {transform-id old-version}}))- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*}))- (testing "Checking that the correct analyses were updated"- (assert-has-analyses- ;; neither unrelated cards nor dependent cards should be updated on transform update- {:card {card-id old-version- other-card-id old-version}- :transform {transform-id new-version}})))))))))+ (testing "Transform update marks entity stale, entity-check job re-analyzes it"+ (let [products (lib.metadata/table mp (mt/id :products))+ old-version models.analysis-finding/*current-analysis-finding-version*]+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query+ :query (lib/query mp products)}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ (deps.findings/upsert-analysis! transform)+ (assert-has-analyses {:transform {transform-id old-version}})+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})+ (testing "Transform should be marked stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :transform+ :analyzed_entity_id transform-id))))+ (#'task.entity-check/check-entities!)+ (testing "Transform should be re-analyzed"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :transform+ :analyzed_entity_id transform-id))))))))))(deftest ^:sequential transform-update-triggers-native-transforms-test(run-with-dependencies-setup!(fn [mp]- (testing "Native transform updates should trigger analysis"- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query- :query (lib/native-query mp "select * from products")}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}]- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (assert-has-analyses- {:transform {transform-id models.analysis-finding/*current-analysis-finding-version*}}))))))+ (testing "Native transform updates do not error"+ (mt/with-temp [:model/Transform transform {:source {:type :query+ :query (lib/native-query mp "select * from products")}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ ;; Should not throw even for native transforms+ (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*}))))))))(deftest ^:sequential transform-run-works-with-no-analyses-test(run-with-dependencies-setup!(fn [mp]- (testing "transform runs work when dependent cards have no pre-existing analyses"- (let [products-id (mt/id :products)- products (lib.metadata/table mp products-id)]+ (testing "Transform run completes without error when no analyses exist"+ (let [products (lib.metadata/table mp (mt/id :products))](mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id} {:source {:type :query- :query (lib/query mp products)}+ :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}:name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :transform- :to_entity_id transform-id}]- ;; Event should complete without error even when there are no pre-existing analyses+ :target {:schema "public" :name "sample" :type :table}}+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :transform :to_entity_id transform-id}](is (some? (events/publish-event! :event/transform-run-complete{:object {:db-id (mt/id):output-schema "public"@@ -1103,152 +845,99 @@:transform-id transform-id}:user-id api/*current-user-id*})))))))))-(deftest ^:sequential transform-run-updates-analyses-test+(deftest ^:sequential transform-run-marks-dependents-stale-test(run-with-dependencies-setup!(fn [mp]- (testing "transform runs update existing analyses when they exist"- (let [products-id (mt/id :products)- orders-id (mt/id :orders)- products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*]+ (testing "Transform run marks immediate dependents stale"+ (let [products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}- :model/Transform {transform-id :id :as transform} {:source {:type :query- :query (lib/query mp products)}+ :model/Transform {transform-id :id :as transform} {:source {:type :query :query (lib/query mp products)}:name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}+ :target {:schema "public" :name "sample" :type :table}}:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :transform- :to_entity_id transform-id}]+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :transform :to_entity_id transform-id}](deps.findings/upsert-analysis! card)(deps.findings/upsert-analysis! transform)(deps.findings/upsert-analysis! other-card)- (testing "checking that initial analyses exist"- (assert-has-analyses- {:card {card-id old-version- other-card-id old-version}- :transform {transform-id old-version}}))- (t2/with-transaction [_conn]- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "public"- :output-table "sample"- :transform-id transform-id}- :user-id api/*current-user-id*})- (testing "dependent card should be marked stale"- (is (=? {:analysis_version old-version :stale true}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id))))- (testing "other card should be unchanged"- (is (=? {:analysis_version old-version :stale false}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id other-card-id))))- (testing "transform should be unchanged"- (is (=? {:analysis_version old-version :stale false}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :transform- :analyzed_entity_id transform-id)))))))))))+ (events/publish-event! :event/transform-run-complete+ {:object {:db-id (mt/id)+ :output-schema "public"+ :output-table "sample"+ :transform-id transform-id}+ :user-id api/*current-user-id*})+ (testing "Dependent card should be marked stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id card-id))))+ (testing "Other card should be unchanged"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id other-card-id))))+ (testing "Transform should be unchanged"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :transform :analyzed_entity_id transform-id))))))))))(deftest ^:sequential segment-update-works-with-no-analyses-test(run-with-dependencies-setup!- (fn [mp]- (testing "segment update creates analysis for the segment itself"+ (fn [_]+ (testing "Segment update does not error when no analyses exist yet"(let [products-id (mt/id :products)- price-field-id (mt/id :products :price)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id- :definition {:filter [:> [:field price-field-id nil] 50]}}- :model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :segment- :to_entity_id segment-id}]- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})- ;; Segment should be analyzed synchronously- (assert-has-analyses- {:segment {segment-id -1}})))))))+ price-field-id (mt/id :products :price)]+ (mt/with-temp [:model/Segment segment {:table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 50]}}]+ ;; Should not throw even when no analysis findings exist+ (is (some? (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})))))))))(deftest ^:sequential segment-update-updates-analyses-test(run-with-dependencies-setup!(fn [mp]- (testing "segment update updates analyses when they exist"+ (testing "Segment update marks entity stale, entity-check job re-analyzes and propagates"(let [products-id (mt/id :products)price-field-id (mt/id :products :price)products (lib.metadata/table mp products-id)- old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]+ old-version models.analysis-finding/*current-analysis-finding-version*](mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id:definition {:filter [:> [:field price-field-id nil] 50]}}:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :segment- :to_entity_id segment-id}]+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :segment :to_entity_id segment-id}](deps.findings/upsert-analysis! segment)(deps.findings/upsert-analysis! card)- (testing "checking that initial analyses exist"- (assert-has-analyses- {:card {card-id old-version}- :segment {segment-id old-version}}))- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (t2/with-transaction [_conn]- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})- (testing "segment should be re-analyzed synchronously"- (is (=? {:analysis_version new-version :stale false}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :segment- :analyzed_entity_id segment-id))))- (testing "dependent card should be marked stale"- (is (=? {:analysis_version old-version :stale true}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id))))))))))))+ (assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})+ (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})+ (testing "Segment should be marked stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :segment :analyzed_entity_id segment-id))))+ (#'task.entity-check/check-entities!)+ (testing "Both segment and dependent card should be re-analyzed"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :segment :analyzed_entity_id segment-id)))+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id card-id))))))))))-;;; ### Table Metadata Update Tests(deftest ^:sequential table-metadata-update-triggers-dependent-analysis-test(run-with-dependencies-setup!(fn [mp]- (testing "Table metadata update triggers re-analysis of dependent cards"+ (testing "Table metadata update marks immediate dependents stale"(let [products-id (mt/id :products)- orders-id (mt/id :orders)+ orders (lib.metadata/table mp (mt/id :orders))products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]+ old-version models.analysis-finding/*current-analysis-finding-version*](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :table- :to_entity_id products-id}]+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}](deps.findings/upsert-analysis! card)(deps.findings/upsert-analysis! other-card)- (testing "checking that initial analyses exist"- (assert-has-analyses- {:card {card-id old-version- other-card-id old-version}}))+ (assert-has-analyses {:card {card-id old-version other-card-id old-version}})(let [table (t2/select-one :model/Table :id products-id)]- (t2/with-transaction [_conn]- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*}))- (testing "card that depends on the updated table should be marked stale"- (is (=? {:analysis_version old-version ; It hasn't been re-analyzed yet- :stale true}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id))))- (testing "card that does not depend on the updated table should not be marked stale"- (is (=? {:analysis_version old-version- :stale false}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id other-card-id))))))))))))+ (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})+ (testing "Card depending on updated table should be marked stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id card-id))))+ (testing "Card not depending on updated table should not be stale"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))(deftest ^:sequential table-metadata-update-works-with-no-analyses-test(run-with-dependencies-setup!@@ -1257,56 +946,35 @@(let [products-id (mt/id :products)products (lib.metadata/table mp products-id)](mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :table- :to_entity_id products-id}]+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}](let [table (t2/select-one :model/Table :id products-id)]- (t2/with-transaction [_conn]- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})- (assert-has-analyses- {:card {card-id nil}})))))))))--;;; ### Field Metadata Update Tests+ (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})+ (assert-has-analyses {:card {card-id nil}}))))))))(deftest ^:sequential field-metadata-update-triggers-dependent-analysis-test(run-with-dependencies-setup!(fn [mp]- (testing "Field metadata update triggers re-analysis of cards depending on the field's table"+ (testing "Field metadata update marks dependents of the field's table stale"(let [products-id (mt/id :products)- orders-id (mt/id :orders)+ orders (lib.metadata/table mp (mt/id :orders))products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)- old-version models.analysis-finding/*current-analysis-finding-version*- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]+ old-version models.analysis-finding/*current-analysis-finding-version*](mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :table- :to_entity_id products-id}]+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}](deps.findings/upsert-analysis! card)(deps.findings/upsert-analysis! other-card)- (testing "checking that initial analyses exist"- (assert-has-analyses- {:card {card-id old-version- other-card-id old-version}}))+ (assert-has-analyses {:card {card-id old-version other-card-id old-version}})(let [field (t2/select-one :model/Field :id (mt/id :products :category))]- (t2/with-transaction [_conn]- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*}))- (testing "card that depends on the table whose field was updated should be marked stale"- (is (=? {:analysis_version old-version ; It hasn't been re-analyzed yet- :stale true}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id))))- (testing "card that does not depend on the table whose field was updated should not be marked stale"- (is (=? {:analysis_version old-version ; It hasn't been re-analyzed yet- :stale false}- (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id other-card-id))))))))))))+ (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})+ (testing "Card depending on the table whose field was updated should be stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id card-id))))+ (testing "Card not depending on that table should not be stale"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))(deftest ^:sequential field-metadata-update-works-with-no-analyses-test(run-with-dependencies-setup!@@ -1315,12 +983,8 @@(let [products-id (mt/id :products)products (lib.metadata/table mp products-id)](mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}- :model/Dependency _ {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :table- :to_entity_id products-id}]+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}](let [field (t2/select-one :model/Field :id (mt/id :products :category))]- (t2/with-transaction [_conn]- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})- (assert-has-analyses- {:card {card-id nil}})))))))))+ (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})+ (assert-has-analyses {:card {card-id nil}}))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.cljindex 72e737ff3a4..cefcd4ba51b 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj@@ -5,6 +5,7 @@[metabase-enterprise.dependencies.analysis :as deps.analysis][metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -128,6 +129,13 @@:analyzed_entity_type :card:analyzed_entity_id [:in card-ids]))+(defn- finding-stale?+ "Returns the stale value for a specific entity's analysis finding, or nil if no finding exists."+ [entity-type entity-id]+ (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type entity-type+ :analyzed_entity_id entity-id))+(deftest ^:sequential analyze-batch-picks-up-missing-analyses-test(testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"(backfill-all-entity-analyses!)@@ -197,3 +205,75 @@(is (= {grandparent-id false, parent-id true, child-id true}(stale-map [grandparent-id parent-id child-id]))"grandparent should NOT be stale, parent and child should be stale (transitive)")))))))))++(deftest ^:sequential wave-propagation-via-entity-check-test+ (testing "analyze-and-propagate! marks immediate dependents stale, and the job loop drains them in waves"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-premium-features #{:dependencies}+ (lib-be/with-metadata-provider-cache+ (mt/with-model-cleanup [:model/AnalysisFinding]+ (mt/with-temp [:model/Card {gp-id :id :as gp-card} {:dataset_query (lib/query mp products)}+ :model/Card {p-id :id :as p-card} {:dataset_query (lib/query mp (lib.metadata/card mp gp-id))}+ :model/Card {c-id :id :as c-card} {:dataset_query (lib/query mp (lib.metadata/card mp p-id))}+ :model/Dependency _ {:from_entity_type :card :from_entity_id p-id+ :to_entity_type :card :to_entity_id gp-id}+ :model/Dependency _ {:from_entity_type :card :from_entity_id c-id+ :to_entity_type :card :to_entity_id p-id}]+ ;; Analyze all three so they have findings+ (run! deps.findings/upsert-analysis! [gp-card p-card c-card])+ (is (= {gp-id false, p-id false, c-id false}+ (stale-map [gp-id p-id c-id]))+ "all should start as not stale")+ ;; Mark only the immediate dependents of grandparent (just parent, not child)+ (deps.findings/mark-immediate-dependents-stale! :card gp-id)+ (is (= {gp-id false, p-id true, c-id false}+ (stale-map [gp-id p-id c-id]))+ "only parent should be stale (immediate), child should NOT be stale yet")+ ;; Run the entity-check job loop — it should analyze parent, which marks child stale,+ ;; then analyze child in the next wave+ (#'task.entity-check/check-entities!)+ (is (= {gp-id false, p-id false, c-id false}+ (stale-map [gp-id p-id c-id]))+ "after check-entities!, all should be re-analyzed (not stale) — wave propagation worked"))))))))++(deftest ^:sequential wave-propagation-through-non-analyzable-entity-test+ (testing "Wave propagation skips non-analyzable intermediaries (e.g., tables) and reaches cards beyond them"+ (backfill-all-entity-analyses!)+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))+ products-id (mt/id :products)]+ (mt/with-premium-features #{:dependencies}+ (lib-be/with-metadata-provider-cache+ (mt/with-model-cleanup [:model/AnalysisFinding]+ ;; Chain: transform → (depends on) table ← (depends on) card+ ;; Table is not analyzable, but card should still be reached by the wave+ (mt/with-temp [:model/Transform {transform-id :id :as transform}+ {:source {:type :query :query (lib/query mp products)}+ :name "test_wave_transform"+ :target {:schema "public" :name "wave_test_table" :type :table}}+ :model/Card {card-id :id :as card}+ {:dataset_query (lib/query mp products)}+ ;; table depends on transform (downstream of transform)+ :model/Dependency _ {:from_entity_type :table :from_entity_id products-id+ :to_entity_type :transform :to_entity_id transform-id}+ ;; card depends on the same table+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id products-id}]+ ;; Analyze the transform and card (tables aren't analyzable)+ (deps.findings/upsert-analysis! transform)+ (deps.findings/upsert-analysis! card)+ (is (false? (finding-stale? :card card-id))+ "card should start as not stale")+ ;; Mark only immediate dependents of transform stale+ ;; The immediate dependent is the table, which is NOT analyzable+ ;; The card is two hops away: transform → table → card+ (deps.findings/mark-immediate-dependents-stale! :transform transform-id)+ ;; The look-through should have reached the card through the non-analyzable table+ (is (true? (finding-stale? :card card-id))+ "card should be stale — look-through reached it via non-analyzable table")+ ;; Run entity-check to drain stale entities+ (#'task.entity-check/check-entities!)+ (is (false? (finding-stale? :card card-id))+ "card should be re-analyzed after entity-check"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.cljindex f4c09abbe98..f7456a0009e 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj@@ -3,6 +3,7 @@[clojure.set :as set][clojure.test :refer [deftest is testing]][metabase-enterprise.dependencies.models.dependency :as deps.graph]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase.graph.core :as graph][metabase.lib.core :as lib][metabase.queries.models.card :as card]@@ -48,8 +49,9 @@(mt/dataset test-data(mt/with-temp [:model/User user {:email "[email protected]"}](mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [card1 (card/create-card! (basic-orders) user)]+ (deps.test/synchronously-run-backfill!)(is (integer? (:id card1)))(testing "when creating a new card"(is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}@@ -57,6 +59,7 @@(testing "that depends on another card"(let [card2 (card/create-card! (wrap-card card1) user)]+ (deps.test/synchronously-run-backfill!)(is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}(upstream-of :card (:id card2))))(testing "but that doesn't affect the upstream deps of the inner card"@@ -71,6 +74,7 @@{:joins [{:alias "Products":source-table (mt/id :products):condition [:= $id &Products.$products.id]}]})}})+ (deps.test/synchronously-run-backfill!)(is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))(depends-on-> :card (:id card1) :table (mt/id :products))}(upstream-of :card (:id card1)))))@@ -78,6 +82,7 @@(card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1)):card-updates{:dataset_query (mt/mbql-query products)}})+ (deps.test/synchronously-run-backfill!)(is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}(upstream-of :card (:id card1)))))))))))))@@ -86,10 +91,11 @@(mt/dataset test-data(mt/with-temp [:model/User user {:email "[email protected]"}](mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [{id1 :id :as card1} (card/create-card! (basic-orders) user){id2 :id :as card2} (card/create-card! (wrap-card card1) user){id3 :id :as card3} (card/create-card! (wrap-card card2) user)]+ (deps.test/synchronously-run-backfill!)(testing "raw deps are recorded correctly"(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))(is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))@@ -124,11 +130,12 @@(mt/dataset test-data(mt/with-temp [:model/User user {:email "[email protected]"}](mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [mp (mt/metadata-provider){id1 :id :as card1} (card/create-card! (sql-card mp "SELECT * FROM orders;") user){id2 :id :as card2} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id1 "}}")) user){id3 :id :as card3} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id2 "}}")) user)]+ (deps.test/synchronously-run-backfill!)(testing "raw deps are recorded correctly"(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))(is (=? #{(depends-on-> :card id2 :table (mt/id :orders))@@ -154,7 +161,7 @@(mt/dataset test-data(mt/with-temp [:model/User user {:email "[email protected]"}](mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [metric-card (card/create-card! {:name "Test Metric":database_id (mt/id):table_id (mt/id :orders)@@ -173,102 +180,95 @@{:aggregation [[:metric (:id metric-card)]]}):visualization_settings {}}user)]+ (deps.test/synchronously-run-backfill!)(testing "raw deps are recorded correctly for question using metric"(is (=? #{(depends-on-> :card (:id question-card) :table (mt/id :orders))(depends-on-> :card (:id question-card) :card (:id metric-card))}(upstream-of :card (:id question-card))))))))))))-(deftest dependency-analysis-version-test- (testing "dependency_analysis_version is updated when entities are created or updated"- (mt/with-empty-h2-app-db!+(deftest dependency-status-marked-stale-test+ (testing "dependency_status is marked stale when entities are created or updated"+ (mt/with-model-cleanup [:model/DependencyStatus](testing "cards"(mt/with-model-cleanup [:model/Dependency]- (mt/with-temp [:model/Card card {:dependency_analysis_version 0}]- (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))+ (mt/with-temp [:model/Card card {}](mt/with-premium-features #{:dependencies}(card/update-card! {:card-before-update card:card-updates {:dataset_query (mt/mbql-query orders)}}))- (is (= deps.graph/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Card (:id card)))))))+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true)))))(testing "transforms create"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/Transform transform]- (is (= deps.graph/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))+ (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))(testing "transforms update"(mt/with-premium-features #{}(mt/with-temp [:model/Transform transform]- (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform))))(mt/with-premium-features #{:dependencies}(t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}}))- (is (= deps.graph/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))+ (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))(testing "snippets create"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/NativeQuerySnippet snippet]- (is (= deps.graph/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))+ (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true)))))(testing "snippets update"(mt/with-premium-features #{}(mt/with-temp [:model/NativeQuerySnippet snippet]- (is (zero? (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet))))(mt/with-premium-features #{:dependencies}(t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))- (is (= deps.graph/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet))))))))))+ (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true))))))))-(deftest dependency-analysis-version-create-card-test- (testing "dependency_analysis_version is updated when a card is created"+(deftest dependency-status-marked-stale-on-card-create-test+ (testing "dependency_status is marked stale when a card is created"(mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user](let [card (card/create-card! (basic-orders) user)]- (is (= deps.graph/current-dependency-analysis-version- (t2/select-one-fn :dependency_analysis_version :model/Card :id (:id card))))))))))+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true))))))))(deftest filtered-graph-dependencies-test(testing "filtered-graph-dependencies respects filter clause"- (mt/with-empty-h2-app-db!- (mt/with-temp [:model/User user {:email "[email protected]"}]- (mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]- (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)- {id2 :id :as card2} (card/create-card! (wrap-card card1) user)- {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]- (testing "without filter, returns all dependencies"- (let [graph (deps.graph/graph-dependencies)- deps (graph/transitive graph [[:card id3]])]- (is (= #{[:card id2] [:card id1] [:table (mt/id :orders)]}- (set deps)))))- (testing "with filter excluding card1, omits card1 and its dependencies"- (let [filter-fn (fn [entity-type-field entity-id-field]- [:and- [:= entity-type-field "card"]- [:in entity-id-field [id2 id3]]])- graph (deps.graph/filtered-graph-dependencies filter-fn)- deps (graph/transitive graph [[:card id3]])]- (is (= #{[:card id2]}- (set deps))- "Should only include card2, not card1 or table")))- (testing "with filter excluding card2, breaks the chain"- (let [filter-fn (fn [entity-type-field entity-id-field]- [:and- [:= entity-type-field "card"]- [:in entity-id-field [id1 id3]]])- graph (deps.graph/filtered-graph-dependencies filter-fn)- deps (graph/transitive graph [[:card id3]])]- (is (= #{}- (set deps))- "Should be empty, chain is broken at card2"))))))))))+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)+ {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]+ (deps.test/synchronously-run-backfill!)+ (testing "without filter, returns all dependencies"+ (let [graph (deps.graph/graph-dependencies)+ deps (graph/transitive graph [[:card id3]])]+ (is (= #{[:card id2] [:card id1] [:table (mt/id :orders)]}+ (set deps)))))+ (testing "with filter excluding card1, omits card1 and its dependencies"+ (let [filter-fn (fn [entity-type-field entity-id-field]+ [:and+ [:= entity-type-field "card"]+ [:in entity-id-field [id2 id3]]])+ graph (deps.graph/filtered-graph-dependencies filter-fn)+ deps (graph/transitive graph [[:card id3]])]+ (is (= #{[:card id2]}+ (set deps))+ "Should only include card2, not card1 or table")))+ (testing "with filter excluding card2, breaks the chain"+ (let [filter-fn (fn [entity-type-field entity-id-field]+ [:and+ [:= entity-type-field "card"]+ [:in entity-id-field [id1 id3]]])+ graph (deps.graph/filtered-graph-dependencies filter-fn)+ deps (graph/transitive graph [[:card id3]])]+ (is (= #{}+ (set deps))+ "Should be empty, chain is broken at card2")))))))))(deftest filtered-graph-dependents-test(testing "filtered-graph-dependents respects filter clause"(mt/with-temp [:model/User user {:email "[email protected]"}](mt/with-premium-features #{:dependencies}- (mt/with-model-cleanup [:model/Card :model/Dependency]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](let [{id1 :id :as card1} (card/create-card! (basic-orders) user){id2 :id :as card2} (card/create-card! (wrap-card card1) user){id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]+ (deps.test/synchronously-run-backfill!)(testing "without filter, returns all dependents"(let [graph (deps.graph/graph-dependents)deps (graph/transitive graph [[:card id1]])]@@ -308,6 +308,7 @@:to_entity_type :table:to_entity_id (:id table1)})(deps.graph/swap-dependency! :card (:id card1) [:table (:id table1)] [:table (:id table2)])+ (deps.test/synchronously-run-backfill!)(is (not (t2/exists? :model/Dependency:from_entity_type :card:from_entity_id (:id card1)@@ -333,6 +334,7 @@:to_entity_id (:id table2)}]);; This would fail with duplicate key error before the fix(deps.graph/swap-dependency! :card (:id card2) [:table (:id table1)] [:table (:id table2)])+ (deps.test/synchronously-run-backfill!)(is (not (t2/exists? :model/Dependency:from_entity_type :card:from_entity_id (:id card2)diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljindex 50f46f7afbb..4d06f647aae 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj@@ -3,8 +3,11 @@[clojure.test :refer :all][environ.core :as env][java-time.api :as t]+ [metabase-enterprise.dependencies.calculation :as deps.calculation][metabase-enterprise.dependencies.models.dependency :as dependencies.model]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status][metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase.events.core :as events][metabase.task.core :as task][metabase.test :as mt]@@ -17,59 +20,76 @@(defn- backfill-all-existing-entities![]- (mt/with-premium-features #{:dependencies}- (while (#'dependencies.backfill/backfill-dependencies!))))+ (deps.test/synchronously-run-backfill!))(defn- backfill-dependencies-single-trigger))+(defn- mark-stale!+ "Mark an entity as stale in dependency_status for testing."+ [entity-type entity-id]+ (deps.dependency-status/mark-stale! entity-type [entity-id]))++(defn- assert-processed+ "Assert that an entity has been processed (not stale, current version)."+ [entity-type entity-id]+ (is (t2/exists? :model/DependencyStatus+ :entity_type entity-type+ :entity_id entity-id+ :stale false+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version)+ (str "Expected " (name entity-type) " " entity-id " to be processed")))+(deftest ^:sequential backfill-dependency-analysis-test- (testing "Test that the backfill job correctly updates the dependency_analysis_version"+ (testing "Test that the backfill job correctly processes stale entities"(backfill-all-existing-entities!)(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")](let [query (mt/mbql-query orders)](mt/with-premium-features #{}- (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}- :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}- :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]- (let [card-count (fn [& args]- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))- current-version dependencies.model/current-dependency-analysis-version]- (is (= 3 (card-count)))+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}+ :model/Card {card2-id :id} {:dataset_query query}+ :model/Card {card3-id :id} {:dataset_query query}]+ ;; Mark all three cards as stale+ (mark-stale! :card card1-id)+ (mark-stale! :card card2-id)+ (mark-stale! :card card3-id)+ (let [stale-count (fn []+ (t2/count :model/DependencyStatus+ :entity_type :card+ :entity_id [:in [card1-id card2-id card3-id]]+ :stale true))]+ (is (= 3 (stale-count)));; first run, should process 2 cards(is (true? (backfill-dependencies-single-trigger!)))- (is (= 2 (card-count :dependency_analysis_version current-version)))+ (is (= 1 (stale-count)));; second run, should process the last card(is (false? (backfill-dependencies-single-trigger!)))- (is (= 3 (card-count :dependency_analysis_version current-version)))+ (is (= 0 (stale-count)));; third run, should not process anything(is (false? (backfill-dependencies-single-trigger!)))- (is (= 3 (card-count :dependency_analysis_version current-version))))))))))+ (is (= 0 (stale-count))))))))))(deftest ^:sequential backfill-transform-test(testing "Test that transforms are correctly backfilled"(backfill-all-existing-entities!)(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")](mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)- :dependency_analysis_version 0}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}:model/Transform {transform-id :id} {:name "Test Transform":source {:type "query":query (mt/mbql-query nil{:source-table (str "card__" card-id)})}- :target {:type "table" :name (mt/random-name)}- :dependency_analysis_version 0}]- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))- (is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))+ :target {:type "table" :name (mt/random-name)}}]+ ;; Mark both as stale+ (mark-stale! :card card-id)+ (mark-stale! :transform transform-id)(is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))(is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))(backfill-dependencies-single-trigger!)- (is (t2/exists? :model/Card :id card-id- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))- (is (t2/exists? :model/Transform :id transform-id- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (assert-processed :card card-id)+ (assert-processed :transform transform-id)(is (t2/exists? :model/Dependency:from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :orders)))@@ -83,8 +103,7 @@(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")](mt/with-premium-features #{}(mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"- :content "SELECT 1"- :dependency_analysis_version 0}+ :content "SELECT 1"}:model/Card {card-id :id} {:dataset_query {:database (mt/id):type :native:native {:query "SELECT * FROM {{my_snippet}}"@@ -95,18 +114,16 @@:display-name "my_snippet":type :snippet:snippet-name "my_snippet"- :snippet-id snippet-id}}}}- :dependency_analysis_version 0}]- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))- (is (t2/exists? :model/NativeQuerySnippet :id snippet-id :dependency_analysis_version 0))+ :snippet-id snippet-id}}}}}]+ ;; Mark both as stale+ (mark-stale! :card card-id)+ (mark-stale! :snippet snippet-id)(is (false? (t2/exists? :model/Dependency:from_entity_type :card :from_entity_id card-id:to_entity_type :snippet :to_entity_id snippet-id)))(backfill-dependencies-single-trigger!)- (is (t2/exists? :model/Card :id card-id- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))- (is (t2/exists? :model/NativeQuerySnippet :id snippet-id- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (assert-processed :card card-id)+ (assert-processed :snippet snippet-id)(is (t2/exists? :model/Dependency:from_entity_type :card :from_entity_id card-id:to_entity_type :snippet :to_entity_id snippet-id)))))))@@ -116,19 +133,16 @@(backfill-all-existing-entities!)(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")](mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)- :dependency_analysis_version 0}]- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (mark-stale! :card card-id)(is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :orders))));; First run(backfill-dependencies-single-trigger!)- (is (t2/exists? :model/Card :id card-id- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (assert-processed :card card-id);; Second run - should not change anything(backfill-dependencies-single-trigger!)- (is (t2/exists? :model/Card :id card-id- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))+ (assert-processed :card card-id)(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :orders))))))))@@ -149,132 +163,220 @@:mb-dependency-backfill-batch-size "1":mb-dependency-backfill-delay-minutes "0":mb-dependency-backfill-variance-minutes "0")]- (let [card-data {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]+ (let [card-data {:dataset_query (mt/mbql-query orders)}](mt/test-helpers-set-global-values!(mt/with-premium-features #{}(mt/with-temp [:model/Card {card1-id :id} card-data:model/Card {card2-id :id} card-data:model/Card {card3-id :id} card-data]- (let [card-count (fn [& args]- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))- current-version dependencies.model/current-dependency-analysis-version]- (is (= 3 (card-count :dependency_analysis_version 0)))+ ;; Mark all as stale+ (mark-stale! :card card1-id)+ (mark-stale! :card card2-id)+ (mark-stale! :card card3-id)+ (let [processed? (fn []+ (= 3 (t2/count :model/DependencyStatus+ :entity_type :card+ :entity_id [:in [card1-id card2-id card3-id]]+ :stale false+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version)))]+ (is (not (processed?)))(mt/with-premium-features #{:dependencies};; Initialize the task, which should schedule the first run(task/init! ::dependencies.backfill/DependencyBackfill)- (wait-for-condition #(= 3 (card-count :dependency_analysis_version current-version)) 10000)- (is (= 3 (card-count :dependency_analysis_version current-version)))))))))))))+ (wait-for-condition processed? 10000)+ (is (processed?))))))))))))++(deftest ^:sequential backfill-error-logging-test+ (testing "When calculate-deps throws, the error is logged and the entity remains stale"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (mark-stale! :card card-id)+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-delay-minutes "0"+ :mb-dependency-backfill-variance-minutes "0")+ deps.calculation/calculate-deps+ (fn [_ _] (throw (ex-info "Simulated error" {})))]+ (backfill-dependencies-single-trigger!))+ ;; Entity should still be stale — not processed, not lost+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))+ ;; No dependencies should have been created+ (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))+ ;; Failure recorded in table+ (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus+ :entity_type :card :entity_id card-id)))))))++(deftest ^:sequential backfill-partial-batch-failure-test+ (testing "A failure on one entity doesn't prevent other entities from being processed"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {good-card-id :id} {:dataset_query (mt/mbql-query orders)}+ :model/Card {bad-card-id :id} {:dataset_query (mt/mbql-query products)}]+ (mark-stale! :card good-card-id)+ (mark-stale! :card bad-card-id)+ (let [original-calculate-deps deps.calculation/calculate-deps]+ (with-redefs [env/env (assoc env/env+ :mb-dependency-backfill-delay-minutes "0"+ :mb-dependency-backfill-variance-minutes "0")+ deps.calculation/calculate-deps+ (fn [entity-type entity]+ (if (= (:id entity) bad-card-id)+ (throw (ex-info "Simulated error" {}))+ (original-calculate-deps entity-type entity)))]+ (backfill-dependencies-single-trigger!)))+ ;; Good card should be processed successfully+ (assert-processed :card good-card-id)+ (is (seq (t2/select :model/Dependency :from_entity_type :card :from_entity_id good-card-id)))+ ;; Bad card should remain stale with failure recorded+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id bad-card-id :stale true))+ (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus+ :entity_type :card :entity_id bad-card-id)))))))(deftest ^:sequential backfill-terminal-failure-test(testing "Entities should be marked as terminally broken after MAX_RETRIES failures"(backfill-all-existing-entities!)(mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (mark-stale! :card card-id)+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- (let [update-attempts (volatile! 0)+ (let [compute-attempts (volatile! 0)failures (inc @#'dependencies.backfill/max-retries)](with-redefs [env/env (assoc env/env:mb-dependency-backfill-delay-minutes "0":mb-dependency-backfill-variance-minutes "0")- t2/update! (fn [model-kw id & args]- (if (and (= model-kw :model/Card)- (= id card-id)- (< @update-attempts failures))- (do- (vswap! update-attempts inc)- (throw (ex-info "Simulated DB error" {:id id})))- (apply t2/update! model-kw id args)))]+ ;; Make compute-deps-for-entity! throw+ deps.calculation/calculate-deps+ (fn [_ _]+ (vswap! compute-attempts inc)+ (throw (ex-info "Simulated computation error" {:card-id card-id})))];; fail MAX_RETRIES + 1 times- (while (< @update-attempts failures)+ (while (< @compute-attempts failures)(backfill-dependencies-single-trigger!))))- ;; verify card is not processed and is terminally broken- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ ;; verify card is still stale (not processed)+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true));; verify subsequent runs don't process it(backfill-dependencies-single-trigger!)- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))))))+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))))))(deftest ^:sequential backfill-delayed-retry-test(testing "Failed entities should be retried after their delay period expires"(backfill-all-existing-entities!)- (let [current-version dependencies.model/current-dependency-analysis-version- update-attempts (volatile! 0)]+ (let [compute-attempts (volatile! 0)](mt/with-premium-features #{}- (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (mark-stale! :card card-id)(with-redefs [env/env (assoc env/env- :mb-dependency-backfill-delay-minutes "1") ; 1 minute delay for retries- t2/update! (fn [model-kw id & args]- (if (and (= model-kw :model/Card)- (= id card-id)- (zero? @update-attempts))- (do- (vswap! update-attempts inc)- (throw (ex-info "Simulated DB error" {:id id})))- (apply t2/update! model-kw id args)))]- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ :mb-dependency-backfill-delay-minutes "1")+ deps.calculation/calculate-deps+ (fn [_ _entity]+ (if (zero? @compute-attempts)+ (do+ (vswap! compute-attempts inc)+ (throw (ex-info "Simulated computation error" {:card-id card-id})))+ ;; Return valid deps on subsequent attempts+ {:table #{(mt/id :orders)}}))]+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true));; first failure - should be put into retry state- ;; fail MAX_RETRIES + 1 times- (while (zero? @update-attempts)+ (while (zero? @compute-attempts)(backfill-dependencies-single-trigger!))- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0)))+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- ;; advance time by less than retry delay - should NOT be processed- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))- (backfill-dependencies-single-trigger!))- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))+ ;; advance time by less than retry delay - should NOT be processed+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))+ (backfill-dependencies-single-trigger!))+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- ;; advance time by more than retry delay - should be processed- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))- (backfill-dependencies-single-trigger!))- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version current-version)))))))+ ;; advance time by more than retry delay - should be processed+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))+ (backfill-dependencies-single-trigger!))+ (assert-processed :card card-id)))))))(deftest backfill-card-does-not-cause-revision-test(testing "backfilling a card does not create a new revision or audit log entry"(backfill-all-existing-entities!)(mt/with-premium-features #{:dependencies :audit-app}- (mt/with-temp [:model/Card {card-id :id- dep-version :dependency_analysis_version}- {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ (mark-stale! :card card-id)(let [revision-count-before (t2/count :model/Revision :model "Card" :model_id card-id)deps-before (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]- (is (= 0 dep-version))(is (= 0 revision-count-before))(is (= 0 deps-before))(backfill-dependencies-single-trigger!)(let [revision-count-after (t2/count :model/Revision :model "Card" :model_id card-id)- dep-version-after (t2/select-one-fn :dependency_analysis_version :model/Card :id card-id)deps-after (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)](is (= 0 revision-count-after))- (is (= dependencies.model/current-dependency-analysis-version dep-version-after))+ (assert-processed :card card-id)(is (= 1 deps-after))))))))(deftest ^:sequential backfill-dependencies-on-serdes-load-test- (testing "Test that serdes load correctly updates the dependency_analysis_version"+ (testing "Serdes load triggers the backfill job (entities remain stale until job runs)"(backfill-all-existing-entities!)(let [query (mt/mbql-query orders)](mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}- :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}- :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]- (let [card-count (fn [& args]- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))- current-version dependencies.model/current-dependency-analysis-version]- (events/publish-event! :event/serdes-load {})- (is (= 3 (card-count :dependency_analysis_version current-version)))))))))+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}+ :model/Card {card2-id :id} {:dataset_query query}+ :model/Card {card3-id :id} {:dataset_query query}]+ ;; Mark all as stale+ (mark-stale! :card card1-id)+ (mark-stale! :card card2-id)+ (mark-stale! :card card3-id)+ ;; serdes-load triggers the async job, not synchronous processing+ (events/publish-event! :event/serdes-load {})+ ;; entities are still stale (job hasn't run yet in this test)+ (is (= 3 (t2/count :model/DependencyStatus+ :entity_type :card+ :entity_id [:in [card1-id card2-id card3-id]]+ :stale true)))+ ;; manually run the backfill to verify it processes them+ (backfill-all-existing-entities!)+ (is (= 3 (t2/count :model/DependencyStatus+ :entity_type :card+ :entity_id [:in [card1-id card2-id card3-id]]+ :stale false+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))(deftest ^:sequential backfill-dependencies-on-token-update-test- (testing "Test that token update correctly updates the dependency_analysis_version"+ (testing "Token update triggers the backfill job (entities remain stale until job runs)"(backfill-all-existing-entities!)(let [query (mt/mbql-query orders)](mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}- :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}- :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]- (let [card-count (fn [& args]- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))- current-version dependencies.model/current-dependency-analysis-version]- (events/publish-event! :event/set-premium-embedding-token {})- (is (= 3 (card-count :dependency_analysis_version current-version)))))))))+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}+ :model/Card {card2-id :id} {:dataset_query query}+ :model/Card {card3-id :id} {:dataset_query query}]+ ;; Mark all as stale+ (mark-stale! :card card1-id)+ (mark-stale! :card card2-id)+ (mark-stale! :card card3-id)+ ;; token update triggers the async job, not synchronous processing+ (events/publish-event! :event/set-premium-embedding-token {})+ ;; entities are still stale (job hasn't run yet in this test)+ (is (= 3 (t2/count :model/DependencyStatus+ :entity_type :card+ :entity_id [:in [card1-id card2-id card3-id]]+ :stale true)))+ ;; manually run the backfill to verify it processes them+ (backfill-all-existing-entities!)+ (is (= 3 (t2/count :model/DependencyStatus+ :entity_type :card+ :entity_id [:in [card1-id card2-id card3-id]]+ :stale false+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))++(deftest ^:sequential backfill-version-outdated-test+ (testing "Entities with outdated version in dependency_status get reprocessed"+ (backfill-all-existing-entities!)+ (mt/with-premium-features #{}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]+ ;; Create a dependency_status entry with an old version (not stale, but outdated)+ (t2/insert! :model/DependencyStatus {:entity_type :card+ :entity_id card-id+ :dependency_analysis_version 0+ :stale false})+ (backfill-dependencies-single-trigger!)+ (assert-processed :card card-id)+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id+ :to_entity_type :table :to_entity_id (mt/id :orders)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.cljindex 29e04d9bbde..42950daa0b1 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj@@ -1,11 +1,21 @@(ns metabase-enterprise.dependencies.test-util(:require[medley.core :as m]+ [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.test-metadata :as meta]- [metabase.lib.test-util.metadata-providers.mock :as providers.mock]))+ [metabase.lib.test-util.metadata-providers.mock :as providers.mock]+ [metabase.test :as mt]))++(defn synchronously-run-backfill!+ "Run the dependency backfill job synchronously, processing all stale/outdated entities+ until there is nothing left to process. Use this in tests after operations that mark+ entities stale (create/update events) and before asserting on dependency data."+ []+ (mt/with-premium-features #{:dependencies}+ (while (#'dependencies.backfill/backfill-dependencies!))))(defn mock-card [metadata-provider {:keys [id query details]}](merge {:lib/type :metadata/carddiff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.cljindex e63f284e70e..d5c37e848b5 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.execute :as replacement.execute][metabase-enterprise.replacement.models.replacement-run :as replacement-run][metabase-enterprise.replacement.protocols :as replacement.protocols]@@ -134,6 +135,7 @@;; Populate dependencies via events(doseq [card [old-model mbql-child-1 mbql-child-2 native-child grandchild grandchild-native]](events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"{:source_entity_id old-id@@ -199,6 +201,7 @@:name "Child card"}](mt/with-model-cleanup [:model/ReplacementRun :model/Dependency](events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!);; Insert a fake active run to simulate one already running(let [run (replacement-run/create-run! :card old-id :card new-id (mt/user->id :crowberto))](replacement-run/start-run! (:id run)))@@ -230,6 +233,7 @@:name "Child Card"}](mt/with-model-cleanup [:model/ReplacementRun :model/Dependency](events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"{:source_entity_id old-id:source_entity_type :card@@ -262,6 +266,7 @@:name "Child card"}](mt/with-model-cleanup [:model/ReplacementRun :model/Dependency](events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"{:source_entity_id old-id:source_entity_type :card@@ -371,6 +376,7 @@;; Populate dependencies via events(doseq [card [model-card question-card]](events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))+ (deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :post 202"ee/replacement/replace-model-with-transform"diff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.cljindex e52e6130f0a..9577079307e 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj@@ -3,6 +3,7 @@(:require[clojure.test :refer [deftest is testing]][metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.protocols :as replacement.protocols][metabase-enterprise.replacement.runner :as replacement.runner][metabase-enterprise.replacement.source-swap :as replacement.source-swap]@@ -196,10 +197,11 @@:dataset_query (lib/query mp (lib.metadata/card mp old-id)):type :question:name "Child Card"}]- (mt/with-model-cleanup [:model/Dependency]+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus];; populate dependencies(events/publish-event! :event/card-create {:object old-card :user-id (mt/user->id :rasta)})(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :rasta)})+ (deps.test/synchronously-run-backfill!)(testing "child card initially points to old model"(is (= old-id (get-in (t2/select-one-fn :dataset_query :model/Card :id child-id)@@ -214,6 +216,7 @@(start-run! [_])(succeed-run! [_])(fail-run! [_ _]))]+ #_{:clj-kondo/ignore [:unresolved-var]}(replacement.runner/run-swap-source! [:card old-id] [:card new-id] progress)(testing "child card's source-card is updated to new model"@@ -254,9 +257,10 @@:dataset_query (lib/query mp (lib.metadata/card mp old-id)):type :question:name "Child 2"}]- (mt/with-model-cleanup [:model/Dependency]+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus](doseq [card [old-card child-1 child-2]](events/publish-event! :event/card-create {:object card :user-id (mt/user->id :rasta)}))+ (deps.test/synchronously-run-backfill!)(let [original-swap! replacement.source-swap/swap-source!](with-redefs [replacement.source-swap/swap-source!diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.cljindex c66e04d9c1f..6ddddbb7fab 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj@@ -3,6 +3,7 @@[clojure.test :refer [deftest is testing]][medley.core :as m][metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.field-refs :as replacement.field-refs][metabase-enterprise.replacement.source-swap :as replacement.source-swap][metabase.events.core :as events]@@ -323,6 +324,7 @@;; Create a dependency: card -> orders table(events/publish-event! :event/card-create {:object (t2/select-one :model/Card card-id):user-id (mt/user->id :rasta)})+ (deps.test/synchronously-run-backfill!)(is (t2/exists? :model/Dependency:from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :orders))@@ -352,6 +354,7 @@(mt/with-model-cleanup [:model/Dependency](events/publish-event! :event/segment-create {:object (t2/select-one :model/Segment segment-id):user-id (mt/user->id :rasta)})+ (deps.test/synchronously-run-backfill!)(is (t2/exists? :model/Dependency:from_entity_type :segment :from_entity_id segment-id:to_entity_type :table :to_entity_id (mt/id :orders)))@@ -381,6 +384,7 @@(mt/with-model-cleanup [:model/Dependency](events/publish-event! :event/measure-create {:object (t2/select-one :model/Measure measure-id):user-id (mt/user->id :rasta)})+ (deps.test/synchronously-run-backfill!)(is (t2/exists? :model/Dependency:from_entity_type :measure :from_entity_id measure-id:to_entity_type :table :to_entity_id (mt/id :orders)))@@ -409,6 +413,7 @@(mt/with-model-cleanup [:model/Dependency](events/publish-event! :event/transform-create {:object (t2/select-one :model/Transform transform-id):user-id (mt/user->id :rasta)})+ (deps.test/synchronously-run-backfill!)(is (t2/exists? :model/Dependency:from_entity_type :transform :from_entity_id transform-id:to_entity_type :table :to_entity_id (mt/id :orders)))diff --git a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.cljindex a066305704a..59f9ad7d421 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer [deftest is testing]][metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.usages :as replacement.usages][metabase.graph.core :as graph][metabase.lib.core :as lib]@@ -78,6 +79,7 @@(mt/with-model-cleanup [:model/Card :model/Dependency](let [source-card (card/create-card! (card-with-query "Source card" :products) user)child-card (card/create-card! (card-sourced-from "Child card" source-card) user)+ _ (deps.test/synchronously-run-backfill!)found-usages (replacement.usages/transitive-usages [:card (:id source-card)])](is (some #(= [:card (:id child-card)] %) found-usages)"Child card should appear in usages"))))))))@@ -91,6 +93,7 @@(let [source-card (card/create-card! (card-with-query "Source card" :products) user)child-card (card/create-card! (card-sourced-from "Child card" source-card) user)grandchild-card (card/create-card! (card-sourced-from "Grandchild card" child-card) user)+ _ (deps.test/synchronously-run-backfill!)found-usages (replacement.usages/transitive-usages [:card (:id source-card)])](is (some #(= [:card (:id child-card)] %) found-usages)"Child card should appear in usages")@@ -104,6 +107,7 @@(let [user (mt/fetch-user :rasta)](mt/with-model-cleanup [:model/Card :model/Dependency](let [lonely-card (card/create-card! (card-with-query "Lonely card" :products) user)+ _ (deps.test/synchronously-run-backfill!)found-usages (replacement.usages/transitive-usages [:card (:id lonely-card)])](is (empty? found-usages)"Card with no dependents should have empty usages"))))))))@@ -115,6 +119,7 @@(let [user (mt/fetch-user :rasta)](mt/with-model-cleanup [:model/Card :model/Dependency](let [card-on-table (card/create-card! (card-with-query "Card on products" :products) user)+ _ (deps.test/synchronously-run-backfill!)found-usages (replacement.usages/transitive-usages [:table (mt/id :products)])](is (some #(= [:card (:id card-on-table)] %) found-usages)"Card using table should appear in usages"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.cljindex eb86588f4cf..11f82ca0059 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.clj@@ -23,8 +23,7 @@:card_id true:table_id true:group_id true- :attribute_remappings {:foo 1}- :dependency_analysis_version 0})+ :attribute_remappings {:foo 1}})(defmacro ^:private with-gtap-cleanup!"Invokes `body` ensuring any `Sandbox` created will be removed afterward. Leaving behind a GTAP candiff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.cljindex 24fb05e57d3..69f533a7a05 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj@@ -52,6 +52,7 @@:model/DataPermissions:model/DatabaseRouter:model/Dependency+ :model/DependencyStatus:model/DocumentBookmark:model/CollectionPermissionGraphRevision:model/DashboardCardSeriesdiff --git a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.cljindex ace7a1f4c2c..cab38ac77d1 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj@@ -3,6 +3,7 @@[clojure.set :as set][clojure.string :as str][clojure.test :refer :all]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.workspaces.common :as ws.common][metabase-enterprise.workspaces.dag-abstract :as dag-abstract][metabase-enterprise.workspaces.execute :as ws.execute]@@ -315,6 +316,7 @@;; Wait for workspace DB initialization and grant all inputs by default.;; Test tables are metadata-only (no physical DB tables), so the isolation layer's GRANT;; fails silently and `access_granted` stays false — we force-grant to avoid that.+ _ (deps.test/synchronously-run-backfill!)_ (when (and ws-id (not (:skip-init workspace)))(ws-done! ws-id)(force-grant-all-inputs! ws-id))]diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.cljindex b4a20c1b746..25e77784aab 100644--- a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj@@ -1,6 +1,7 @@(ns ^:mb/driver-tests metabase-enterprise.workspaces.validation-test(:require[clojure.test :refer :all]+ [metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.workspaces.impl :as ws.impl][metabase-enterprise.workspaces.test-util :as ws.tu][metabase-enterprise.workspaces.validation :as ws.validation]@@ -90,6 +91,7 @@:database (mt/id):schema "public":name "external_output"}}]+ (deps.test/synchronously-run-backfill!);; Create a workspace transform that targets the same table(let [{ref-id :ref_id} (mt/user-http-request :crowberto :post 200(str "ee/workspace/" ws-id "/transform")diff --git a/resources/migrations/060/20260402_dependency_status.yaml b/resources/migrations/060/20260402_dependency_status.yamlnew file mode 100644index 00000000000..bcc59af98ca--- /dev/null+++ b/resources/migrations/060/20260402_dependency_status.yaml@@ -0,0 +1,458 @@+databaseChangeLog:++ # 1. Create the dependency_status table+ - changeSet:+ id: v60.2026-04-02T00:00:00+ author: ericnormand+ comment: Create dependency_status table for async dependency calculation tracking+ changes:+ - createTable:+ tableName: dependency_status+ remarks: Tracks dependency calculation status for entities, analogous to analysis_finding+ columns:+ - column:+ name: id+ type: int+ autoIncrement: true+ constraints:+ primaryKey: true+ nullable: false+ - column:+ name: entity_type+ type: varchar(254)+ remarks: The type of the entity (card, transform, snippet, etc.)+ constraints:+ nullable: false+ - column:+ name: entity_id+ type: int+ remarks: The ID of the entity+ constraints:+ nullable: false+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ remarks: Version of the dependency analysis that was performed+ constraints:+ nullable: false+ - column:+ name: stale+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the entity needs dependency recalculation+ constraints:+ nullable: false+ - column:+ name: fail_count+ type: int+ defaultValueNumeric: 0+ remarks: Number of consecutive failed dependency calculation attempts+ constraints:+ nullable: false+ - column:+ name: next_retry_at+ type: ${timestamp_type}+ remarks: Earliest time at which a failed entity can be retried (null means immediately processable)+ - column:+ name: terminal+ type: ${boolean.type}+ defaultValueBoolean: false+ remarks: True when the entity has exceeded max retries and should not be retried until modified+ constraints:+ nullable: false+ rollback:+ - dropTable:+ tableName: dependency_status++ # 2. Unique constraint on (entity_type, entity_id)+ - changeSet:+ id: v60.2026-04-02T00:00:01+ author: ericnormand+ comment: Unique constraint to prevent duplicate dependency_status entries+ changes:+ - addUniqueConstraint:+ tableName: dependency_status+ columnNames: entity_type, entity_id+ constraintName: idx_unique_dependency_status+ rollback:+ - dropUniqueConstraint:+ tableName: dependency_status+ constraintName: idx_unique_dependency_status++ # 3. Index on stale column (partial on postgres, full on others)+ - changeSet:+ id: v60.2026-04-02T00:00:02+ author: ericnormand+ comment: Add partial index on dependency_status.stale for efficient polling+ changes:+ - sql:+ dbms: postgresql+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale) WHERE stale = true+ - sql:+ dbms: mysql,mariadb,h2+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale)+ rollback:+ - dropIndex:+ tableName: dependency_status+ indexName: idx_dependency_status_stale++ # 4-11. Migrate data from entity tables to dependency_status+ # All entities are migrated. Entities with version > 0 (already analyzed) get stale=false.+ # Entities with version = 0 (never analyzed) get stale=true so the backfill picks them up.++ - changeSet:+ id: v60.2026-04-02T00:00:03+ author: ericnormand+ comment: Migrate dependency_analysis_version from report_card to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'card', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM report_card+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'card'++ - changeSet:+ id: v60.2026-04-02T00:00:04+ author: ericnormand+ comment: Migrate dependency_analysis_version from transform to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'transform', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM transform+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'transform'++ - changeSet:+ id: v60.2026-04-02T00:00:05+ author: ericnormand+ comment: Migrate dependency_analysis_version from native_query_snippet to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'snippet', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM native_query_snippet+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'snippet'++ - changeSet:+ id: v60.2026-04-02T00:00:06+ author: ericnormand+ comment: Migrate dependency_analysis_version from report_dashboard to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'dashboard', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM report_dashboard+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'dashboard'++ - changeSet:+ id: v60.2026-04-02T00:00:07+ author: ericnormand+ comment: Migrate dependency_analysis_version from document to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'document', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM document+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'document'++ - changeSet:+ id: v60.2026-04-02T00:00:08+ author: ericnormand+ comment: Migrate dependency_analysis_version from sandboxes to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'sandbox', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM sandboxes+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'sandbox'++ - changeSet:+ id: v60.2026-04-02T00:00:09+ author: ericnormand+ comment: Migrate dependency_analysis_version from segment to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'segment', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM segment+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'segment'++ - changeSet:+ id: v60.2026-04-02T00:00:10+ author: ericnormand+ comment: Migrate dependency_analysis_version from measure to dependency_status+ changes:+ - sql:+ sql: >-+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)+ SELECT 'measure', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM measure+ rollback:+ - sql:+ sql: DELETE FROM dependency_status WHERE entity_type = 'measure'++ # 12-17. Drop indexes on dependency_analysis_version (for tables that have them)++ - changeSet:+ id: v60.2026-04-02T00:00:11+ author: ericnormand+ comment: Drop dependency_analysis_version index from report_card+ changes:+ - dropIndex:+ tableName: report_card+ indexName: idx_report_card_dependency_analysis_version+ rollback:+ - createIndex:+ tableName: report_card+ indexName: idx_report_card_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version++ - changeSet:+ id: v60.2026-04-02T00:00:12+ author: ericnormand+ comment: Drop dependency_analysis_version index from transform+ changes:+ - dropIndex:+ tableName: transform+ indexName: idx_transform_dependency_analysis_version+ rollback:+ - createIndex:+ tableName: transform+ indexName: idx_transform_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version++ - changeSet:+ id: v60.2026-04-02T00:00:13+ author: ericnormand+ comment: Drop dependency_analysis_version index from native_query_snippet+ changes:+ - dropIndex:+ tableName: native_query_snippet+ indexName: idx_native_query_snippet_dependency_analysis_version+ rollback:+ - createIndex:+ tableName: native_query_snippet+ indexName: idx_native_query_snippet_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version++ - changeSet:+ id: v60.2026-04-02T00:00:14+ author: ericnormand+ comment: Drop dependency_analysis_version index from report_dashboard+ changes:+ - dropIndex:+ tableName: report_dashboard+ indexName: idx_report_dashboard_dependency_analysis_version+ rollback:+ - createIndex:+ tableName: report_dashboard+ indexName: idx_report_dashboard_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version++ - changeSet:+ id: v60.2026-04-02T00:00:15+ author: ericnormand+ comment: Drop dependency_analysis_version index from document+ changes:+ - dropIndex:+ tableName: document+ indexName: idx_document_dependency_analysis_version+ rollback:+ - createIndex:+ tableName: document+ indexName: idx_document_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version++ - changeSet:+ id: v60.2026-04-02T00:00:16+ author: ericnormand+ comment: Drop dependency_analysis_version index from sandboxes+ changes:+ - dropIndex:+ tableName: sandboxes+ indexName: idx_sandboxes_dependency_analysis_version+ rollback:+ - createIndex:+ tableName: sandboxes+ indexName: idx_sandboxes_dependency_analysis_version+ columns:+ - column:+ name: dependency_analysis_version++ # 18-25. Drop dependency_analysis_version columns from all 8 entity tables++ - changeSet:+ id: v60.2026-04-02T00:00:17+ author: ericnormand+ comment: Drop dependency_analysis_version column from report_card+ changes:+ - dropColumn:+ tableName: report_card+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: report_card+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ constraints:+ nullable: false++ - changeSet:+ id: v60.2026-04-02T00:00:18+ author: ericnormand+ comment: Drop dependency_analysis_version column from transform+ changes:+ - dropColumn:+ tableName: transform+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: transform+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ constraints:+ nullable: false++ - changeSet:+ id: v60.2026-04-02T00:00:19+ author: ericnormand+ comment: Drop dependency_analysis_version column from native_query_snippet+ changes:+ - dropColumn:+ tableName: native_query_snippet+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: native_query_snippet+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ constraints:+ nullable: false++ - changeSet:+ id: v60.2026-04-02T00:00:20+ author: ericnormand+ comment: Drop dependency_analysis_version column from report_dashboard+ changes:+ - dropColumn:+ tableName: report_dashboard+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: report_dashboard+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ constraints:+ nullable: false++ - changeSet:+ id: v60.2026-04-02T00:00:21+ author: ericnormand+ comment: Drop dependency_analysis_version column from document+ changes:+ - dropColumn:+ tableName: document+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: document+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ constraints:+ nullable: false++ - changeSet:+ id: v60.2026-04-02T00:00:22+ author: ericnormand+ comment: Drop dependency_analysis_version column from sandboxes+ changes:+ - dropColumn:+ tableName: sandboxes+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: sandboxes+ columns:+ - column:+ name: dependency_analysis_version+ type: smallint+ defaultValue: 0+ constraints:+ nullable: false++ - changeSet:+ id: v60.2026-04-02T00:00:23+ author: ericnormand+ comment: Drop dependency_analysis_version column from segment+ changes:+ - dropColumn:+ tableName: segment+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: segment+ columns:+ - column:+ name: dependency_analysis_version+ type: integer+ defaultValueNumeric: 0+ constraints:+ nullable: false++ - changeSet:+ id: v60.2026-04-02T00:00:24+ author: ericnormand+ comment: Drop dependency_analysis_version column from measure+ changes:+ - dropColumn:+ tableName: measure+ columnName: dependency_analysis_version+ rollback:+ - addColumn:+ tableName: measure+ columns:+ - column:+ name: dependency_analysis_version+ type: integer+ defaultValueNumeric: 0+ constraints:+ nullable: falsediff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex 07e6933c7a1..3d385d706ec 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -402,8 +402,7 @@:skip [;; those stats are inherently local state:view_count :last_viewed_at;; this is deprecated- :cache_ttl- :dependency_analysis_version]+ :cache_ttl]:transform {:created_at (serdes/date):initially_published_at (serdes/date):collection_id (serdes/fk :model/Collection)diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.cljindex fe10f85de7e..3d4b5a0d42f 100644--- a/src/metabase/documents/models/document.clj+++ b/src/metabase/documents/models/document.clj@@ -192,7 +192,7 @@(defmethod serdes/make-spec "Document"[_model-name _opts]{:copy [:archived :archived_directly :content_type :entity_id :name :collection_position]- :skip [:view_count :last_viewed_at :public_uuid :made_public_by_id :dependency_analysis_version]+ :skip [:view_count :last_viewed_at :public_uuid :made_public_by_id]:transform {:created_at (serdes/date):updated_at (serdes/date):document {:export-with-context export-document-contentdiff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.cljindex d457c2ffffd..83e0b59d1f3 100644--- a/src/metabase/measures/models/measure.clj+++ b/src/metabase/measures/models/measure.clj@@ -208,8 +208,7 @@(defmethod serdes/make-spec "Measure" [_model-name _opts]{:copy [:name :archived :description :entity_id]- :skip [:dependency_analysis_version- ;; dimensions are computed from the query and reconciled on read, not serialized+ :skip [;; dimensions are computed from the query and reconciled on read, not serialized:dimensions :dimension_mappings]:transform {:created_at (serdes/date):table_id (serdes/fk :model/Table)diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.cljindex 1dac8950d36..68209f8ae52 100644--- a/src/metabase/models/resolution.clj+++ b/src/metabase/models/resolution.clj@@ -42,6 +42,7 @@:model/Database metabase.warehouses.models.database:model/DatabaseRouter metabase-enterprise.database-routing.models:model/Dependency metabase-enterprise.dependencies.models.dependency+ :model/DependencyStatus metabase-enterprise.dependencies.models.dependency-status:model/Dimension metabase.warehouse-schema.models.dimension:model/Document metabase.documents.models.document:model/DocumentBookmark metabase.bookmarks.models.bookmarkdiff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex 72e8d132f70..22cf75d276c 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -173,7 +173,7 @@(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]{:copy [:archived :content :description :entity_id :name :template_tags]- :skip [:dependency_analysis_version]+ :skip []:transform {:created_at (serdes/date):collection_id (serdes/fk :model/Collection):creator_id (serdes/fk :model/User)}diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex e1f1b9dc9f3..520ab07ceb3 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -1320,8 +1320,6 @@:dataset_query_metrics_v2_migration_backup;; this column is not used anymore:cache_ttl- ;; dependencies aren't serialized, so the version of dependency analysis done shouldn't be serialized- :dependency_analysis_version;; dimensions are computed from the query and reconciled on read, not serialized:dimensions :dimension_mappings;; temporary column to power rollback from v57 to v56; we can remove it in v58diff --git a/src/metabase/revisions/impl/dashboard.clj b/src/metabase/revisions/impl/dashboard.cljindex c71d00a9cce..b66cb754039 100644--- a/src/metabase/revisions/impl/dashboard.clj+++ b/src/metabase/revisions/impl/dashboard.clj@@ -18,7 +18,6 @@;; > The position this Dashboard should appear in the Dashboards list,;; lower-numbered positions appearing before higher numbered ones.;; TODO: querying on stats we don't have any dashboard that has a position, maybe we could just drop it?- :dependency_analysis_version:public_uuid :made_public_by_id:position :initially_published_at :view_count:last_viewed_at})diff --git a/src/metabase/revisions/impl/measure.clj b/src/metabase/revisions/impl/measure.cljindex f2523bf42c3..dab753ecf92 100644--- a/src/metabase/revisions/impl/measure.clj+++ b/src/metabase/revisions/impl/measure.clj@@ -5,7 +5,7 @@[metabase.revisions.models.revision :as revision]))(def ^:private excluded-columns-for-measure-revision- #{:created_at :updated_at :dependency_analysis_version})+ #{:created_at :updated_at})(defmethod revision/serialize-instance :model/Measure[_model _id instance]diff --git a/src/metabase/revisions/impl/segment.clj b/src/metabase/revisions/impl/segment.cljindex f1dbde9f4c8..e734b2ba257 100644--- a/src/metabase/revisions/impl/segment.clj+++ b/src/metabase/revisions/impl/segment.clj@@ -5,7 +5,7 @@[metabase.revisions.models.revision :as revision]))(def ^:private excluded-columns-for-segment-revision- #{:created_at :updated_at :dependency_analysis_version})+ #{:created_at :updated_at})(defmethod revision/serialize-instance :model/Segment[_model _id instance]diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex 8a201b3fe63..6c6eb59ec25 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -227,7 +227,7 @@(defmethod serdes/make-spec "Segment" [_model-name _opts]{:copy [:name :points_of_interest :archived :caveats :description :entity_id :show_in_getting_started]- :skip [:dependency_analysis_version]+ :skip []:transform {:created_at (serdes/date):table_id (serdes/fk :model/Table):creator_id (serdes/fk :model/User)diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex 88fcfa8a847..3a747d21218 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -359,7 +359,7 @@(defmethod serdes/make-spec "Transform"[_model-name opts]{:copy [:name :description :entity_id :owner_email]- :skip [:dependency_analysis_version :source_type :target_db_id :target_table_id :last_checkpoint_value]+ :skip [:source_type :target_db_id :target_table_id :last_checkpoint_value]:transform {:created_at (serdes/date):creator_id (serdes/fk :model/User):owner_user_id (serdes/fk :model/User)diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 9931dac6558..94bdc358fc3 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -86,7 +86,6 @@[:collection_id [:maybe pos-int?]][:target_db_id {:optional true} [:maybe pos-int?]][:run_trigger {:optional true} [:maybe :keyword]]- [:dependency_analysis_version :int][:creator CreatorResponse][:last_run {:optional true} [:maybe TransformLastRunResponse]][:tag_ids {:optional true} [:sequential pos-int?]]diff --git a/src/metabase/transforms_rest/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.cljindex 92ca02ad8e4..8329e05de58 100644--- a/src/metabase/transforms_rest/api/transform_job.clj+++ b/src/metabase/transforms_rest/api/transform_job.clj@@ -79,7 +79,6 @@[:creator_id pos-int?][:collection_id [:maybe pos-int?]][:run_trigger {:optional true} [:maybe :keyword]]- [:dependency_analysis_version :int][:creator CreatorResponse]])(api.macros/defendpoint :post "/" :- TransformJobResponsediff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.cljindex ac9a9b43cd2..1e3731916d4 100644--- a/test/metabase/cmd/copy_test.clj+++ b/test/metabase/cmd/copy_test.clj@@ -36,6 +36,7 @@:model/DashboardFavorite:model/DatabaseRouter:model/Dependency+ :model/DependencyStatus:model/MetabotGroupLimit:model/MetabotInstanceLimit:model/MetabotPermissionsdiff --git a/test/metabase/documents/models/document_test.clj b/test/metabase/documents/models/document_test.cljindex a081c27ebd5..47860692519 100644--- a/test/metabase/documents/models/document_test.clj+++ b/test/metabase/documents/models/document_test.clj@@ -458,7 +458,7 @@(let [spec (serdes/make-spec "Document" {})](is (= [:archived :archived_directly :content_type :entity_id :name :collection_position](:copy spec)))- (is (= [:view_count :last_viewed_at :public_uuid :made_public_by_id :dependency_analysis_version] (:skip spec)))+ (is (= [:view_count :last_viewed_at :public_uuid :made_public_by_id] (:skip spec)))(is (contains? (:transform spec) :created_at))(is (contains? (:transform spec) :document))(is (contains? (:transform spec) :updated_at))diff --git a/test/metabase/revisions/impl/card_test.clj b/test/metabase/revisions/impl/card_test.cljindex c69042ffd99..5db31dab234 100644--- a/test/metabase/revisions/impl/card_test.clj+++ b/test/metabase/revisions/impl/card_test.clj@@ -139,9 +139,6 @@;; we don't expect a description for this column because it should never change;; once created by the migration:dataset_query_metrics_v2_migration_backup- ;; `dependency_analysis_version` is an internal bookkeeping field. It doesn't affect the- ;; actual card itself, so no description is necessary.- :dependency_analysis_version;; this column is immutable in practice, and doesn't warrant a description:workspace_id}col)diff --git a/test/metabase/revisions/impl/segment_test.clj b/test/metabase/revisions/impl/segment_test.cljindex 4787c226c09..04a07394fb6 100644--- a/test/metabase/revisions/impl/segment_test.clj+++ b/test/metabase/revisions/impl/segment_test.clj@@ -28,7 +28,7 @@(update :id boolean)(update :table_id boolean)))))(testing "excluded columns are not present"- (is (not-any? #{:created_at :updated_at :dependency_analysis_version}+ (is (not-any? #{:created_at :updated_at}(keys (revision/serialize-instance :model/Segment (:id segment) segment)))))))(deftest ^:parallel diff-segments-testdiff --git a/test/metabase/segments/api_test.clj b/test/metabase/segments/api_test.cljindex 2232af82cf5..f8de61f85ce 100644--- a/test/metabase/segments/api_test.clj+++ b/test/metabase/segments/api_test.clj@@ -21,7 +21,7 @@(defn- segment-response [segment](-> (into {} segment)- (dissoc :id :table_id :dependency_analysis_version)+ (dissoc :id :table_id)(update :creator #(into {} %))(update :entity_id some?)(update :created_at some?)