Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GHY-3232: Make dependency graph calculation async on entity updates (#71037)

Eric Normand5d1d7d4a49bd
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 912c61dcb8b..1e993e18095 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2843,7 +2843,6 @@
api
app-db
collections
- config
documents
driver
events
@@ -2868,8 +2867,6 @@
:model-imports #{:model/Card
:model/Collection
:model/Dashboard
- :model/DashboardCard
- :model/DashboardCardSeries
:model/Document
:model/Field
:model/Measure
diff --git a/e2e/support/helpers/e2e-dependency-helpers.ts b/e2e/support/helpers/e2e-dependency-helpers.ts
index cba03fcb949..04f39db9b80 100644
--- a/e2e/support/helpers/e2e-dependency-helpers.ts
+++ b/e2e/support/helpers/e2e-dependency-helpers.ts
@@ -1,5 +1,8 @@
import type {
+ DependencyGraph as DependencyGraphData,
+ DependencyId,
DependencyNode,
+ DependencyType,
ListBreakingGraphNodesResponse,
ListUnreferencedGraphNodesResponse,
} from "metabase-types/api";
@@ -42,9 +45,56 @@ export const DependencyDiagnostics = {
},
};
-const WAIT_TIMEOUT = 10000;
+const WAIT_TIMEOUT = 30000;
const WAIT_INTERVAL = 100;
+export function waitForBackfillComplete(
+ timeout = WAIT_TIMEOUT,
+): Cypress.Chainable {
+ return cy
+ .request<{
+ complete: boolean;
+ }>("GET", "/api/ee/dependencies/backfill-status")
+ .then((response) => {
+ if (response.body.complete) {
+ return cy.wrap(response);
+ } else if (timeout > 0) {
+ cy.wait(WAIT_INTERVAL);
+ return waitForBackfillComplete(timeout - WAIT_INTERVAL);
+ } else {
+ throw new Error("Dependency backfill timeout");
+ }
+ });
+}
+
+export function waitForGraphDependencies(
+ id: DependencyId,
+ type: DependencyType,
+ filter: (graph: DependencyGraphData) => boolean,
+ timeout = WAIT_TIMEOUT,
+): Cypress.Chainable {
+ return cy
+ .request<DependencyGraphData>(
+ "GET",
+ `/api/ee/dependencies/graph?id=${id}&type=${type}`,
+ )
+ .then((response) => {
+ if (filter(response.body)) {
+ return cy.wrap(response);
+ } else if (timeout > 0) {
+ cy.wait(WAIT_INTERVAL);
+ return waitForGraphDependencies(
+ id,
+ type,
+ filter,
+ timeout - WAIT_INTERVAL,
+ );
+ } else {
+ throw new Error("Dependency graph retry timeout");
+ }
+ });
+}
+
export function waitForUnreferencedEntities(
filter: (nodes: DependencyNode[]) => boolean,
timeout = WAIT_TIMEOUT,
diff --git a/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.ts b/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.ts
index 8d6568cc73a..88e89d2a0ce 100644
--- a/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/data-studio-metrics.cy.spec.ts
@@ -102,6 +102,8 @@ describe("scenarios > data studio > library > metrics", () => {
H.echartsContainer().findByText("Count").should("be.visible");
cy.log("Verify metric dependencies page");
+ H.waitForBackfillComplete();
+ cy.reload();
H.DataStudio.Metrics.dependenciesTab().click();
H.DependencyGraph.graph().within(() => {
cy.findByText("Orders").should("be.visible");
diff --git a/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.ts b/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.ts
index 3dcb4c71ab8..ffa3721f34b 100644
--- a/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/data-studio-tables.cy.spec.ts
@@ -160,7 +160,7 @@ describe("scenarios > data studio > library > tables", () => {
name: "Test question",
query: { "source-table": ORDERS_ID },
});
-
+ H.waitForBackfillComplete();
H.DataStudio.Tables.visitOverviewPage(ORDERS_ID);
H.DataStudio.Tables.dependenciesTab().click();
H.DependencyGraph.graph().within(() => {
diff --git a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
index 7b93e9952db..dcc33bd244e 100644
--- a/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/transforms/transforms.cy.spec.ts
@@ -1924,6 +1924,7 @@ LIMIT
visitTransform: true,
});
+ H.waitForBackfillComplete();
H.DataStudio.Transforms.dependenciesTab().click();
H.DataStudio.Dependencies.content()
.should("contain", "Transform B")
diff --git a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts
index 617db43985f..2f2d69182bd 100644
--- a/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts
+++ b/e2e/test/scenarios/dependencies/dependency-graph.cy.spec.ts
@@ -690,6 +690,8 @@ function visitGraph() {
}
function visitGraphForEntity(id: DependencyId, type: DependencyType) {
+ // Wait for async dependency computation to complete before navigating
+ H.waitForBackfillComplete();
return cy.visit(BASE_URL, { qs: { id, type } });
}
diff --git a/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.ts b/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.ts
index 10db442122b..813854fa17a 100644
--- a/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.ts
+++ b/e2e/test/scenarios/dependencies/dependency-unreferenced-list.cy.spec.ts
@@ -120,6 +120,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
describe("analysis", () => {
it("should show unreferenced entities", () => {
setupEntities();
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
H.DependencyDiagnostics.list().within(() => {
ENTITY_NAMES.forEach((name) => {
@@ -142,6 +143,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
describe("search", () => {
it("should search for entities", () => {
setupEntities();
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
H.DependencyDiagnostics.searchInput().type(
MODEL_FOR_QUESTION_DATA_SOURCE,
@@ -154,6 +156,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
it("should search for entities with type filters", () => {
setupEntities();
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
H.DependencyDiagnostics.searchInput().type("tag");
checkList({
@@ -176,6 +179,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
describe("filters", () => {
it("should filter entities by type", () => {
setupEntities();
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
checkList({ visibleEntities: ENTITY_NAMES });
@@ -206,9 +210,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
it("should persist filter changes after page reload", () => {
setupEntities();
- H.waitForUnreferencedEntities((entities) =>
- entities.some((e) => e.data.name === MODEL_FOR_QUESTION_DATA_SOURCE),
- );
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
checkList({ visibleEntities: MODEL_NAMES });
@@ -222,9 +224,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
it("should filter by location", () => {
setupEntities();
- H.waitForUnreferencedEntities((entities) =>
- entities.some((e) => e.data.name === MODEL_FOR_MODEL_DATA_SOURCE),
- );
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
checkList({
visibleEntities: [
@@ -258,6 +258,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
describe("sorting", () => {
it("should sort by name", () => {
setupEntities();
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
H.DependencyDiagnostics.searchInput().type("Model for");
@@ -281,6 +282,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
it("should sort by location", () => {
setupEntities();
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
H.DependencyDiagnostics.searchInput().type("Model for");
@@ -299,6 +301,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
it("should persist sorting changes after page reload", () => {
setupEntities();
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
H.DependencyDiagnostics.searchInput().type("Model for");
@@ -314,12 +317,7 @@ describe("scenarios > dependencies > unreferenced list", () => {
describe("selecting entities", () => {
it("should show the sidebar for supported entities and trigger snowplow event", () => {
setupEntities();
- H.waitForUnreferencedEntities((entities) =>
- entities.some(
- (e) =>
- e.type === "table" && e.data.display_name === TABLE_DISPLAY_NAME,
- ),
- );
+ H.waitForBackfillComplete();
H.DependencyDiagnostics.visitUnreferencedEntities();
H.DependencyDiagnostics.list().findByText(TABLE_DISPLAY_NAME).click();
diff --git a/e2e/test/scenarios/metrics/metric-page.cy.spec.ts b/e2e/test/scenarios/metrics/metric-page.cy.spec.ts
index 159b06074b0..9b3fb2443a7 100644
--- a/e2e/test/scenarios/metrics/metric-page.cy.spec.ts
+++ b/e2e/test/scenarios/metrics/metric-page.cy.spec.ts
@@ -376,6 +376,7 @@ describe("scenarios > metrics > metric page", () => {
it("should show the Dependencies tab with dependency graph in EE", () => {
H.createQuestion(ORDERS_SCALAR_METRIC).then(({ body: metric }) => {
+ H.waitForBackfillComplete();
H.visitMetric(metric.id);
});
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 4c1d35f95a0..7c4e295f690 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -7,6 +7,7 @@
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
[metabase-enterprise.dependencies.models.analysis-finding-error :as analysis-finding-error]
[metabase-enterprise.dependencies.models.dependency :as dependency]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
[metabase.analyze.core :as analyze]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
@@ -1187,6 +1188,13 @@
(fetch-and-hydrate-nodes nodes-by-type))
(sort-dependents sort-column sort-direction)))))
+(api.macros/defendpoint :get "/backfill-status" :- [:map
+ [:complete :boolean]]
+ "Returns whether the dependency backfill has pending work.
+ `complete` is true when there are no stale or outdated entities awaiting processing."
+ [_route-params _query-params]
+ {:complete (not (deps.dependency-status/has-stale-or-outdated?))})
+
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/dependencies` routes."
(handlers/routes
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
index 41260a6060a..03fa2bbac71 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -6,12 +6,26 @@
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
- [metabase.queries.schema :as queries.schema]
[metabase.transforms-base.util :as transforms-base.u]
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.malli :as mu]))
+(defmulti calculate-deps*
+ "Implementation multimethod for [[calculate-deps]]. Dispatches on entity-type keyword.
+ Prefer calling [[calculate-deps]] which validates the return value."
+ {:arglists '([entity-type entity])}
+ (fn [entity-type _entity] entity-type))
+
+(mu/defn calculate-deps :- ::deps.schema/upstream-deps
+ "Calculate upstream dependencies for a single entity.
+ Returns a map of dependency-type -> set of entity IDs."
+ [entity-type :- keyword?
+ entity]
+ (calculate-deps* entity-type entity))
+
+;;; ------------------------------------------------ Helpers ------------------------------------------------
+
(mu/defn- upstream-deps:mbql-query :- ::deps.schema/upstream-deps
[query :- ::lib.schema/query]
{:card (or (lib/all-source-card-ids query) #{})
@@ -34,9 +48,32 @@
(upstream-deps:native-query query)
(upstream-deps:mbql-query query)))
-(mu/defn upstream-deps:card :- ::deps.schema/upstream-deps
- "Given a Toucan `:model/Card`, return its upstream dependencies as a map from the kind to a set of IDs."
- [{query :dataset_query :as card} :- ::queries.schema/card]
+(mu/defn- upstream-deps:python-transform :- ::deps.schema/upstream-deps
+ [{{tables :source-tables} :source :as _py-transform}
+ :- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]
+ {:table (into #{} (keep :table_id) tables)})
+
+;; Modified implementation of documents.models.document/document-deps
+(defn- document-deps
+ [{:keys [content_type] :as document}]
+ (when (= content_type prose-mirror/prose-mirror-content-type)
+ (prose-mirror/collect-ast document (fn [{:keys [type attrs]}]
+ (cond
+ (and (= prose-mirror/smart-link-type type)
+ (#{"card" "dashboard" "table" "document"} (:model attrs)))
+ [(keyword (:model attrs)) (:entityId attrs)]
+
+ (= prose-mirror/card-embed-type type)
+ [:card (:id attrs)]
+
+ :else
+ nil)))))
+
+;;; ------------------------------------------------ defmethods ------------------------------------------------
+
+(defmethod calculate-deps* :card
+ [_ {query :dataset_query :as card}]
+ {:pre [(some? query)]}
(let [query-deps (upstream-deps:query query)
param-card-ids (keep #(-> % :values_source_config :card_id) (:parameters card))]
(reduce (fn [deps card-id]
@@ -44,21 +81,8 @@
query-deps
param-card-ids)))
-(mu/defn upstream-deps:python-transform :- ::deps.schema/upstream-deps
- "Given a Toucan `:model/Transform`, return its upstream dependencies as a map from the kind to a set of IDs."
- [{{tables :source-tables} :source :as _py-transform}
- :- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]
- {:table (into #{} (keep :table_id) tables)})
-
-(mu/defn upstream-deps:transform :- ::deps.schema/upstream-deps
- "Given a Transform (in Toucan form), return its upstream dependencies."
- [{{:keys [query]} :source :as transform} :-
- [:map
- [:source [:multi {:dispatch (comp keyword :type)}
- [:query
- [:map [:query ::lib.schema/query]]]
- [:python
- [:map [:source-tables {:optional true} [:sequential ::transforms-base.u/source-table-entry]]]]]]]]
+(defmethod calculate-deps* :transform
+ [_ {{:keys [query]} :source :as transform}]
(let [source-type (transforms-base.u/transform-type transform)]
(case source-type
:query (upstream-deps:query query)
@@ -66,9 +90,8 @@
(do (log/warnf "Don't know how to analyze the deps of Transform %d with source type '%s'" (:id transform) source-type)
{}))))
-(mu/defn upstream-deps:snippet :- ::deps.schema/upstream-deps
- "Given a native query snippet, return its upstream dependencies in the usual `{entity-type #{1 2 3}}` format."
- [{:keys [template_tags] :as _snippet}]
+(defmethod calculate-deps* :snippet
+ [_ {:keys [template_tags] :as _snippet}]
(let [type->id-key {:card :card-id, :snippet :snippet-id}
dependencies (keep (fn [tag]
(let [entity-type (:type tag)]
@@ -78,10 +101,10 @@
(vals template_tags))]
(u/group-by first second conj #{} dependencies)))
-(mu/defn upstream-deps:dashboard :- ::deps.schema/upstream-deps
- "Given a dashboard, return its upstream dependencies"
- [{:keys [dashcards series-card-ids] :as dashboard}]
+(defmethod calculate-deps* :dashboard
+ [_ {:keys [dashcards] :as dashboard}]
(let [card-ids (into #{} (keep :card_id dashcards))
+ series-card-ids (into #{} (comp (mapcat :series) (map :id)) dashcards)
param-card-ids (into #{} (keep (comp :card_id :values_source_config) (:parameters dashboard)))
vis-setting-target-ids (fn [link-type]
(into #{} (keep (fn [dashcard]
@@ -111,47 +134,27 @@
{:card all-card-ids
:dashboard all-dashboard-ids}))
-;; Modified implementation of documents.models.document/document-deps
-(defn- document-deps
- [{:keys [content_type] :as document}]
- (when (= content_type prose-mirror/prose-mirror-content-type)
- (prose-mirror/collect-ast document (fn [{:keys [type attrs]}]
- (cond
- (and (= prose-mirror/smart-link-type type)
- (#{"card" "dashboard" "table" "document"} (:model attrs)))
- [(keyword (:model attrs)) (:entityId attrs)]
-
- (= prose-mirror/card-embed-type type)
- [:card (:id attrs)]
-
- :else
- nil)))))
-
-(mu/defn upstream-deps:document :- ::deps.schema/upstream-deps
- "Given a document, return its upstream dependencies"
- [document]
+(defmethod calculate-deps* :document
+ [_ document]
(reduce (fn [deps [dep-type dep-id]]
(update deps dep-type (fnil conj #{}) dep-id))
{}
(document-deps document)))
-(mu/defn upstream-deps:sandbox :- ::deps.schema/upstream-deps
- "Given a sandbox, return its upstream dependencies"
- [sandbox]
+(defmethod calculate-deps* :sandbox
+ [_ sandbox]
(if-let [card-id (:card_id sandbox)]
{:card #{card-id}}
{}))
-(mu/defn upstream-deps:segment :- ::deps.schema/upstream-deps
- "Given a segment, return its upstream dependencies (the table it filters and any segments it references)"
- [{:keys [table_id definition] :as _segment}]
+(defmethod calculate-deps* :segment
+ [_ {:keys [table_id definition] :as _segment}]
{:segment (or (lib/all-segment-ids definition) #{})
:table (cond-> (into #{} (lib/all-implicitly-joined-table-ids definition))
table_id (conj table_id))})
-(mu/defn upstream-deps:measure :- ::deps.schema/upstream-deps
- "Given a measure, return its upstream dependencies (the table it aggregates, any measures it references, and any segments it references)"
- [{:keys [table_id definition] :as _measure}]
+(defmethod calculate-deps* :measure
+ [_ {:keys [table_id definition] :as _measure}]
{:measure (or (lib/all-measure-ids definition) #{})
:segment (or (lib/all-segment-ids definition) #{})
:table (cond-> (into #{} (lib/all-implicitly-joined-table-ids definition))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
index 8f38a363ef0..98a90d90b9b 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
@@ -1,14 +1,12 @@
(ns metabase-enterprise.dependencies.events
(:require
- [metabase-enterprise.dependencies.async :as async]
- [metabase-enterprise.dependencies.calculation :as deps.calculation]
[metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
+ [metabase-enterprise.dependencies.task.backfill :as task.backfill]
[metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
[metabase.events.core :as events]
- [metabase.lib-be.core :as lib-be]
[metabase.premium-features.core :as premium-features]
- [metabase.transforms.core :as transforms]
[metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -17,42 +15,27 @@
;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the
;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created
;; or changed at that point.
-
-(defmacro ignore-errors
- "Ignore errors.
-
- In practice, we cannot reliably distinguish permanent and temporary errors, so in principle
- every error should be retried a few times. Unfortunately that doesn't work, because updating the
- dependency_analysis_version field itself is a trigger for new analysis, so the caller has no
- way to give up and commit the new version after a few retries. We stop on every error until this
- becomes a serious enough issue, at which point we will have to redesign version marking and
- analysis triggering."
- {:style/indent 0}
- [entity-type entity-id & body]
- `(try
- ~@body
- (catch Throwable e#
- (log/error e# "Dependency calculation failed" {:entity-type ~entity-type
- :entity-id ~entity-id})
- nil)))
+;;
+;; Create/update handlers mark entities stale in dependency_status. The backfill task does the actual computation.
+
+(defn- mark-stale-and-trigger!
+ "Mark an entity as stale in dependency_status and trigger the backfill job."
+ [entity-type entity-id]
+ (try
+ (deps.dependency-status/mark-stale! entity-type [entity-id])
+ (task.backfill/trigger-backfill-job!)
+ (catch Throwable e
+ (log/error e "Failed to mark entity stale" {:entity-type entity-type :entity-id entity-id}))))
;; ### Cards
(derive ::card-deps :metabase/event)
(derive :event/card-create ::card-deps)
(derive :event/card-update ::card-deps)
-;; Backfill-only event that triggers dependency calculation without creating a revision (#66365)
-(derive :event/card-dependency-backfill ::card-deps)
(methodical/defmethod events/publish-event! ::card-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (models.dependency/replace-dependencies! :card (:id object)
- (ignore-errors :card (:id object)
- (deps.calculation/upstream-deps:card object)))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/Card (:id object)
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))
+ (mark-stale-and-trigger! :card (:id object))))
(derive ::card-delete :metabase/event)
(derive :event/card-delete ::card-delete)
@@ -60,7 +43,8 @@
(methodical/defmethod events/publish-event! ::card-delete
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))))
+ (t2/delete! :model/Dependency :from_entity_type :card :from_entity_id (:id object))
+ (t2/delete! :model/DependencyStatus :entity_type :card :entity_id (:id object))))
;; ### Snippets
(derive ::snippet-deps :metabase/event)
@@ -70,13 +54,7 @@
(methodical/defmethod events/publish-event! ::snippet-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (models.dependency/replace-dependencies! :snippet (:id object)
- (ignore-errors :snippet (:id object)
- (deps.calculation/upstream-deps:snippet object)))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/NativeQuerySnippet (:id object)
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))
+ (mark-stale-and-trigger! :snippet (:id object))))
(derive ::snippet-delete :metabase/event)
(derive :event/snippet-delete ::snippet-delete)
@@ -84,48 +62,18 @@
(methodical/defmethod events/publish-event! ::snippet-delete
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))))
+ (t2/delete! :model/Dependency :from_entity_type :snippet :from_entity_id (:id object))
+ (t2/delete! :model/DependencyStatus :entity_type :snippet :entity_id (:id object))))
;; ### Transforms
(derive ::transform-deps :metabase/event)
(derive :event/create-transform ::transform-deps)
(derive :event/update-transform ::transform-deps)
-;; On *saving* a transform, the upstream deps of its query are computed and saved.
-(defn- drop-outdated-target-dep! [{:keys [id target] :as transform}]
- (let [db-id (transforms/transform-source-database transform)
- downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency
- :from_entity_type :table
- :to_entity_type :transform
- :to_entity_id id)
- downstream-tables (when (seq downstream-table-ids)
- (t2/select :model/Table :id [:in downstream-table-ids]))
- outdated-tables (remove (fn [table]
- (and (= (:schema table) (:schema target))
- (= (:name table) (:name target))
- (or (not db-id)
- (= db-id (:db_id table)))))
- downstream-tables)
- not-found-table-ids (remove (into #{} (map :id) downstream-tables)
- downstream-table-ids)]
- (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)
- (map :id) outdated-tables))]
- (t2/delete! :model/Dependency
- :from_entity_type :table
- :from_entity_id [:in outdated-downstream-table-ids]
- :to_entity_type :transform
- :to_entity_id id))))
-
(methodical/defmethod events/publish-event! ::transform-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (models.dependency/replace-dependencies! :transform (:id object)
- (ignore-errors :transform (:id object)
- (deps.calculation/upstream-deps:transform object)))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/Transform (:id object) {:dependency_analysis_version models.dependency/current-dependency-analysis-version}))
- (drop-outdated-target-dep! object))))
+ (mark-stale-and-trigger! :transform (:id object))))
(derive ::transform-delete :metabase/event)
(derive :event/delete-transform ::transform-delete)
@@ -134,7 +82,8 @@
[_ {:keys [id]}]
;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?
(when (premium-features/has-feature? :dependencies)
- (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)))
+ (t2/delete! :model/Dependency :from_entity_type :transform :from_entity_id id)
+ (t2/delete! :model/DependencyStatus :entity_type :transform :entity_id id)))
;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.
;; (And if the target has changed, the old table's dep on the transform is dropped.)
@@ -157,25 +106,11 @@
(derive ::dashboard-deps :metabase/event)
(derive :event/dashboard-create ::dashboard-deps)
(derive :event/dashboard-update ::dashboard-deps)
-;; Backfill-only event that triggers dependency calculation without creating a revision
-(derive :event/dashboard-dependency-backfill ::dashboard-deps)
(methodical/defmethod events/publish-event! ::dashboard-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (let [dashboard-id (:id object)
- dashcards (t2/select :model/DashboardCard :dashboard_id dashboard-id)
- series-card-ids (when (seq dashcards)
- (t2/select-fn-set :card_id :model/DashboardCardSeries
- :dashboardcard_id [:in (map :id dashcards)]))
- dashboard (assoc object :dashcards dashcards :series-card-ids series-card-ids)]
- (models.dependency/replace-dependencies! :dashboard dashboard-id
- (ignore-errors :dashboard dashboard-id
- (deps.calculation/upstream-deps:dashboard dashboard))))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/Dashboard (:id object)
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))
+ (mark-stale-and-trigger! :dashboard (:id object))))
(derive ::dashboard-delete :metabase/event)
(derive :event/dashboard-delete ::dashboard-delete)
@@ -183,7 +118,8 @@
(methodical/defmethod events/publish-event! ::dashboard-delete
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/delete! :model/Dependency :from_entity_type :dashboard :from_entity_id (:id object))))
+ (t2/delete! :model/Dependency :from_entity_type :dashboard :from_entity_id (:id object))
+ (t2/delete! :model/DependencyStatus :entity_type :dashboard :entity_id (:id object))))
;; ### Documents
(derive ::document-deps :metabase/event)
@@ -193,13 +129,7 @@
(methodical/defmethod events/publish-event! ::document-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (models.dependency/replace-dependencies! :document (:id object)
- (ignore-errors :document (:id object)
- (deps.calculation/upstream-deps:document object)))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/Document (:id object)
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))
+ (mark-stale-and-trigger! :document (:id object))))
(derive ::document-delete :metabase/event)
(derive :event/document-delete ::document-delete)
@@ -207,7 +137,8 @@
(methodical/defmethod events/publish-event! ::document-delete
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/delete! :model/Dependency :from_entity_type :document :from_entity_id (:id object))))
+ (t2/delete! :model/Dependency :from_entity_type :document :from_entity_id (:id object))
+ (t2/delete! :model/DependencyStatus :entity_type :document :entity_id (:id object))))
;; ### Sandboxes
(derive ::sandbox-deps :metabase/event)
@@ -217,12 +148,7 @@
(methodical/defmethod events/publish-event! ::sandbox-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (models.dependency/replace-dependencies! :sandbox (:id object) (ignore-errors :sandbox (:id object)
- (deps.calculation/upstream-deps:sandbox object)))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/Sandbox (:id object)
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))
+ (mark-stale-and-trigger! :sandbox (:id object))))
(derive ::sandbox-delete :metabase/event)
(derive :event/sandbox-delete ::sandbox-delete)
@@ -230,7 +156,8 @@
(methodical/defmethod events/publish-event! ::sandbox-delete
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/delete! :model/Dependency :from_entity_type :sandbox :from_entity_id (:id object))))
+ (t2/delete! :model/Dependency :from_entity_type :sandbox :from_entity_id (:id object))
+ (t2/delete! :model/DependencyStatus :entity_type :sandbox :entity_id (:id object))))
;; ### Segments
(derive ::segment-deps :metabase/event)
@@ -240,20 +167,16 @@
(methodical/defmethod events/publish-event! ::segment-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (models.dependency/replace-dependencies! :segment (:id object)
- (ignore-errors :segment (:id object)
- (deps.calculation/upstream-deps:segment object)))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/Segment (:id object)
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))
+ (mark-stale-and-trigger! :segment (:id object))))
(derive ::segment-delete :metabase/event)
(derive :event/segment-delete ::segment-delete)
(methodical/defmethod events/publish-event! ::segment-delete
[_ {:keys [object]}]
- (t2/delete! :model/Dependency :from_entity_type :segment :from_entity_id (:id object)))
+ (when (premium-features/has-feature? :dependencies)
+ (t2/delete! :model/Dependency :from_entity_type :segment :from_entity_id (:id object))
+ (t2/delete! :model/DependencyStatus :entity_type :segment :entity_id (:id object))))
;; ### Measures
(derive ::measure-deps :metabase/event)
@@ -263,13 +186,7 @@
(methodical/defmethod events/publish-event! ::measure-deps
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/with-transaction [_conn]
- (models.dependency/replace-dependencies! :measure (:id object)
- (ignore-errors :measure (:id object)
- (deps.calculation/upstream-deps:measure object)))
- (when (not= (:dependency_analysis_version object) models.dependency/current-dependency-analysis-version)
- (t2/update! :model/Measure (:id object)
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version})))))
+ (mark-stale-and-trigger! :measure (:id object))))
(derive ::measure-delete :metabase/event)
(derive :event/measure-delete ::measure-delete)
@@ -277,50 +194,76 @@
(methodical/defmethod events/publish-event! ::measure-delete
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))))
+ (t2/delete! :model/Dependency :from_entity_type :measure :from_entity_id (:id object))
+ (t2/delete! :model/DependencyStatus :entity_type :measure :entity_id (:id object))))
+
+;; ## Checking dependents for breakage (analysis_finding staleness)
+;;
+;; This is a SEPARATE staleness system from the dependency_status table above.
+;; - dependency_status.stale: tracks whether an entity's upstream *dependency graph* needs recomputation.
+;; Handled by the backfill job (task.backfill).
+;; - analysis_finding staleness: tracks whether an entity's dependents need re-analysis for *breakage detection*
+;; (e.g., a model column was removed — are downstream cards broken?). Handled by the entity-check job
+;; (task.entity-check).
+;;
+;; Both are triggered from the same entity events but serve different purposes and run independently.
(derive ::check-card-dependents :metabase/event)
(derive :event/card-create ::check-card-dependents)
(derive :event/card-update ::check-card-dependents)
-(derive :event/card-delete ::check-card-dependents)
(methodical/defmethod events/publish-event! ::check-card-dependents
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (async/submit!
- (fn []
- (lib-be/with-metadata-provider-cache
- (let [has-stale-dependents? (t2/with-transaction [_conn]
- (deps.findings/upsert-analysis! object)
- (deps.findings/mark-dependents-stale! :card (:id object)))]
- (when has-stale-dependents?
- (task.entity-check/trigger-entity-check-job!))))))))
+ (deps.findings/mark-entity-stale! :card (:id object))
+ (task.entity-check/trigger-entity-check-job!)))
+
+(derive ::check-card-dependents-on-delete :metabase/event)
+(derive :event/card-delete ::check-card-dependents-on-delete)
+
+(methodical/defmethod events/publish-event! ::check-card-dependents-on-delete
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (when (deps.findings/mark-immediate-dependents-stale! :card (:id object))
+ (task.entity-check/trigger-entity-check-job!))))
(derive ::check-transform :metabase/event)
(derive :event/create-transform ::check-transform)
(derive :event/update-transform ::check-transform)
-(derive :event/delete-transform ::check-transform)
(methodical/defmethod events/publish-event! ::check-transform
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! object))))
+ (deps.findings/mark-entity-stale! :transform (:id object))
+ (task.entity-check/trigger-entity-check-job!)))
+
+(derive ::check-transform-on-delete :metabase/event)
+(derive :event/delete-transform ::check-transform-on-delete)
+
+(methodical/defmethod events/publish-event! ::check-transform-on-delete
+ [_ {:keys [id]}]
+ (when (premium-features/has-feature? :dependencies)
+ (when (deps.findings/mark-immediate-dependents-stale! :transform id)
+ (task.entity-check/trigger-entity-check-job!))))
(derive ::check-segment-dependents :metabase/event)
(derive :event/segment-create ::check-segment-dependents)
(derive :event/segment-update ::check-segment-dependents)
-(derive :event/segment-delete ::check-segment-dependents)
(methodical/defmethod events/publish-event! ::check-segment-dependents
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (lib-be/with-metadata-provider-cache
- (let [has-stale-dependents? (t2/with-transaction [_conn]
- (deps.findings/upsert-analysis! object)
- (deps.findings/mark-dependents-stale! :segment (:id object)))]
- (when has-stale-dependents?
- (task.entity-check/trigger-entity-check-job!))))))
+ (deps.findings/mark-entity-stale! :segment (:id object))
+ (task.entity-check/trigger-entity-check-job!)))
+
+(derive ::check-segment-dependents-on-delete :metabase/event)
+(derive :event/segment-delete ::check-segment-dependents-on-delete)
+
+(methodical/defmethod events/publish-event! ::check-segment-dependents-on-delete
+ [_ {:keys [object]}]
+ (when (premium-features/has-feature? :dependencies)
+ (when (deps.findings/mark-immediate-dependents-stale! :segment (:id object))
+ (task.entity-check/trigger-entity-check-job!))))
(derive ::check-transform-dependents :metabase/event)
(derive :event/transform-run-complete ::check-transform-dependents)
@@ -328,7 +271,7 @@
(methodical/defmethod events/publish-event! ::check-transform-dependents
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (when (deps.findings/mark-dependents-stale! :transform (:transform-id object))
+ (when (deps.findings/mark-immediate-dependents-stale! :transform (:transform-id object))
(task.entity-check/trigger-entity-check-job!))))
(defn- synced-db->direct-dependents-of-changed-tables
@@ -374,7 +317,7 @@
(when (premium-features/has-feature? :dependencies)
(let [changes (synced-db->direct-dependents-of-changed-tables db-id)]
(when (and (seq changes)
- (deps.findings/mark-all-dependents-stale! {:table changes}))
+ (deps.findings/mark-all-immediate-dependents-stale! {:table changes}))
(task.entity-check/trigger-entity-check-job!)))))
;; ### Admin UI Table/Field Metadata Updates
@@ -385,7 +328,8 @@
(methodical/defmethod events/publish-event! ::check-table-metadata-update
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (deps.findings/mark-dependents-stale! :table (:id object))))
+ (when (deps.findings/mark-immediate-dependents-stale! :table (:id object))
+ (task.entity-check/trigger-entity-check-job!))))
(derive ::check-field-metadata-update :metabase/event)
(derive :event/field-update ::check-field-metadata-update)
@@ -393,4 +337,5 @@
(methodical/defmethod events/publish-event! ::check-field-metadata-update
[_ {:keys [object]}]
(when (premium-features/has-feature? :dependencies)
- (deps.findings/mark-dependents-stale! :table (:table_id object))))
+ (when (deps.findings/mark-immediate-dependents-stale! :table (:table_id object))
+ (task.entity-check/trigger-entity-check-job!))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
index 116d79b2f10..c0282ae94e9 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
@@ -76,17 +76,31 @@
(into [:enum] dependable-entities))
(defn- mark-supported-dependents-stale!
- "Given a map of {entity-type [entity-ids]}, mark all supported types as stale.
+ "Given a map of {entity-type [entity-ids]}, mark all supported (analyzable) types as stale.
+ For non-analyzable entities (e.g., tables), looks through to their dependents so the wave
+ can reach analyzable entities beyond them.
Returns true if any supported dependents were found."
[dependents]
- (let [supported-dependents (into {}
- (filter (fn [[dep-type dep-ids]]
- (and (analyzable-entities dep-type)
- (seq dep-ids))))
- dependents)]
- (doseq [[dep-type dep-ids] supported-dependents]
+ (let [{analyzable true non-analyzable false}
+ (group-by (fn [[dep-type _]] (boolean (analyzable-entities dep-type)))
+ (filter (fn [[_ dep-ids]] (seq dep-ids)) dependents))
+ ;; For non-analyzable entities, find their direct dependents
+ pass-through-dependents
+ (when (seq non-analyzable)
+ (let [key-seq (for [[dep-type dep-ids] non-analyzable
+ id dep-ids]
+ [dep-type id])
+ deps-map (models.dependency/direct-dependents key-seq)]
+ (models.dependency/group-nodes (into #{} cat (vals deps-map)))))
+ ;; Merge the analyzable dependents with the pass-through dependents
+ all-supported (merge-with into
+ (into {} analyzable)
+ (into {}
+ (filter (fn [[dep-type _]] (analyzable-entities dep-type)))
+ pass-through-dependents))]
+ (doseq [[dep-type dep-ids] all-supported]
(deps.analysis-finding/mark-stale! dep-type dep-ids))
- (boolean (seq supported-dependents))))
+ (boolean (seq all-supported))))
(mu/defn mark-dependents-stale! :- :boolean
"Mark all transitive dependents of an entity as stale for re-analysis.
@@ -100,15 +114,50 @@
(let [dependents (models.dependency/transitive-dependents {entity-type [{:id entity-id}]})]
(mark-supported-dependents-stale! dependents)))
-(defn mark-all-dependents-stale!
- "Mark all transitive dependents of multiple entities as stale.
+(defn mark-all-immediate-dependents-stale!
+ "Mark the immediate dependents of multiple entities as stale.
Takes a map of {entity-type entity-ids} where entity-ids is a set of IDs.
+ Does NOT traverse transitively — the entity-check job loop propagates in waves.
Returns true if any supported dependents were found."
[type->entity-ids]
- (let [type->objects (update-vals type->entity-ids #(map (fn [id] {:id id}) %))
- dependents (models.dependency/transitive-dependents type->objects)]
+ (let [key-seq (for [[entity-type ids] type->entity-ids
+ id ids]
+ [entity-type id])
+ deps-map (models.dependency/direct-dependents key-seq)
+ dependents (models.dependency/group-nodes
+ (into #{} cat (vals deps-map)))]
(mark-supported-dependents-stale! dependents)))
+(mu/defn mark-entity-stale!
+ "Mark a single entity as stale for re-analysis by the background job."
+ [entity-type :- AnalyzableEntityType
+ entity-id :- pos-int?]
+ (deps.analysis-finding/mark-stale! entity-type [entity-id]))
+
+(mu/defn mark-immediate-dependents-stale! :- :boolean
+ "Mark the immediate dependents of an entity as stale.
+ Unlike [[mark-dependents-stale!]], this does NOT traverse transitively."
+ [entity-type :- DependableEntityType
+ entity-id :- pos-int?]
+ (let [key-seq [[entity-type entity-id]]
+ deps-map (models.dependency/direct-dependents key-seq)
+ dependents (models.dependency/group-nodes
+ (into #{} cat (vals deps-map)))]
+ (mark-supported-dependents-stale! dependents)))
+
+(defn- analyze-and-propagate!
+ "Analyze an entity and mark its immediate dependents as stale.
+ Wrapped in a transaction so that if marking dependents fails, the analysis
+ is rolled back and the entity stays stale for retry on the next pass.
+ The newly-stale dependents are picked up by the job's loop in
+ `task.entity-check/check-entities!`, which drains all stale entities
+ before returning — no need to re-trigger the job."
+ [instance]
+ (let [entity-type (deps.dependency-types/model->dependency-type (t2/model instance))]
+ (t2/with-transaction [_conn]
+ (upsert-analysis! instance)
+ (mark-immediate-dependents-stale! entity-type (:id instance)))))
+
(mu/defn analyze-batch! :- nat-int?
"Add or update analyses for a batch of entities.
@@ -118,5 +167,9 @@
batch-size :- pos-int?]
(let [instances (deps.analysis-finding/instances-for-analysis type batch-size)]
(lib-be/with-metadata-provider-cache
- (analyze-instances! instances))
+ (doseq [instance instances]
+ (try (analyze-and-propagate! instance)
+ (catch Exception e
+ (log/errorf e "Analyzing entity %s %s failed"
+ (t2/model instance) (:id instance))))))
(count instances)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
new file mode 100644
index 00000000000..ade8ddfa90a
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
@@ -0,0 +1,131 @@
+(ns metabase-enterprise.dependencies.models.dependency-status
+ (:require
+ [java-time.api :as t]
+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase.app-db.core :as app-db]
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/DependencyStatus [_model] :dependency_status)
+
+(derive :model/DependencyStatus :metabase/model)
+
+(t2/deftransforms :model/DependencyStatus
+ {:entity_type mi/transform-keyword})
+
+(defn mark-stale!
+ "Mark entities of `entity-type` with ids in `entity-ids` as stale for dependency recalculation.
+ Creates entries if they don't exist, or sets stale=true if they do.
+ Resets retry state so previously-failed entities get a fresh chance.
+ Uses [[app-db/update-or-insert!]] for cross-database atomicity."
+ [entity-type entity-ids]
+ (doseq [id entity-ids]
+ (app-db/update-or-insert!
+ :model/DependencyStatus
+ {:entity_type entity-type :entity_id id}
+ (fn [existing]
+ (if existing
+ {:stale true :fail_count 0 :next_retry_at nil :terminal false}
+ {:stale true :dependency_analysis_version 0})))))
+
+(defn upsert-status!
+ "Upsert a dependency_status entry, setting stale=false, version to current,
+ and clearing any failure state. Uses [[app-db/update-or-insert!]] for cross-database atomicity."
+ [entity-type entity-id]
+ (app-db/update-or-insert!
+ :model/DependencyStatus
+ {:entity_type entity-type :entity_id entity-id}
+ (fn [_existing]
+ {:dependency_analysis_version models.dependency/current-dependency-analysis-version
+ :stale false
+ :fail_count 0
+ :next_retry_at nil})))
+
+(defmulti hydrate-for-deps
+ "Hydrate a batch of instances with data needed for dependency calculation.
+ Dispatches on entity-type keyword. Default is identity (no hydration needed)."
+ {:arglists '([entity-type instances])}
+ (fn [entity-type _instances] entity-type))
+
+(defmethod hydrate-for-deps :default [_ instances] instances)
+
+(defmethod hydrate-for-deps :dashboard [_ instances]
+ (t2/hydrate instances [:dashcards :series]))
+
+(defn instances-for-dependency-calculation
+ "Find a batch of instances of type `entity-type` and maximum size `batch-size` that need
+ dependency calculation: stale=true OR version < current.
+ Excludes terminal entities and entities whose retry delay hasn't elapsed.
+ Returns full entity objects. Prioritizes stale over outdated.
+ Uses Java time (not DB time) so tests with [[mt/with-clock]] work correctly."
+ [entity-type batch-size]
+ (let [model (deps.dependency-types/dependency-type->model entity-type)
+ table-name (t2/table-name model)
+ id-field (keyword (name table-name) "id")
+ table-wildcard (keyword (name table-name) "*")
+ now (t/offset-date-time)]
+ (t2/select model
+ {:select [table-wildcard]
+ :from table-name
+ :inner-join [:dependency_status [:and
+ [:= :dependency_status.entity_id id-field]
+ [:= :dependency_status.entity_type (name entity-type)]]]
+ :where [:and
+ ;; Needs processing: stale or version outdated
+ [:or
+ [:= :dependency_status.stale true]
+ [:< :dependency_status.dependency_analysis_version
+ models.dependency/current-dependency-analysis-version]]
+ ;; Not terminally broken
+ [:= :dependency_status.terminal false]
+ ;; Retry delay has elapsed (or no delay set)
+ [:or
+ [:is :dependency_status.next_retry_at nil]
+ [:<= :dependency_status.next_retry_at now]]]
+ :order-by [[[:case [:= :dependency_status.stale true] [:inline 0] :else [:inline 1]]]]
+ :limit batch-size})))
+
+(defn has-pending-retries?
+ "Returns true if there are any entities waiting to be retried (not terminal, with a set retry time)."
+ []
+ (t2/exists? :model/DependencyStatus
+ :terminal false
+ :next_retry_at [:not= nil]))
+
+(defn has-stale-or-outdated?
+ "Returns true if there are any entities needing dependency calculation:
+ stale=true OR version < current, not terminal, and retry delay elapsed."
+ []
+ (let [now (t/offset-date-time)]
+ (t2/exists? :model/DependencyStatus
+ {:where [:and
+ [:or
+ [:= :stale true]
+ [:< :dependency_analysis_version
+ models.dependency/current-dependency-analysis-version]]
+ [:= :terminal false]
+ [:or
+ [:is :next_retry_at nil]
+ [:<= :next_retry_at now]]]})))
+
+(defn record-failure!
+ "Record a failed dependency calculation attempt for an entity.
+ Increments fail_count and sets next_retry_at based on exponential backoff.
+ If max retries exceeded, marks the entity as terminal."
+ [entity-type entity-id max-retries delay-minutes]
+ (when-let [status (t2/select-one :model/DependencyStatus
+ :entity_type entity-type
+ :entity_id entity-id)]
+ (let [new-fail-count (inc (:fail_count status 0))]
+ (if (> new-fail-count max-retries)
+ (t2/update! :model/DependencyStatus (:id status)
+ {:fail_count new-fail-count
+ :terminal true
+ :next_retry_at nil})
+ (let [retry-minutes (* new-fail-count delay-minutes)]
+ (t2/update! :model/DependencyStatus (:id status)
+ {:fail_count new-fail-count
+ :next_retry_at (when (pos? retry-minutes)
+ (t/plus (t/offset-date-time) (t/minutes retry-minutes)))}))))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
index 1463e08f3f8..425337090a8 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
@@ -3,157 +3,139 @@
(see [[metabase-enterprise.dependencies.models.dependency]]) up to date, that is, makes sure
the dependency table contains fresh entries.
- This is done by finding entities whose dependency_analysis_version is less than
- [[metabase-enterprise.dependencies.models.dependency/current-dependency-analysis-version]] and
- updates this field. In most cases this triggers an event handler that calculates the dependencies
- and populates the dependency table. For cards, the event is emitted by the job handler itself,
- because the update of the record doesn't trigger the event handler. "
+ This is done by querying the dependency_status table for entities that are stale or have an
+ outdated dependency_analysis_version. The backfill task computes dependencies and updates
+ the dependency_status table."
(:require
- [java-time.api :as t]
+ [metabase-enterprise.dependencies.calculation :as deps.calculation]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
[metabase-enterprise.dependencies.settings :as deps.settings]
[metabase-enterprise.dependencies.task-util :as deps.task-util]
- [metabase.config.core :as config]
[metabase.events.core :as events]
[metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
- [metabase.util :as u]
+ [metabase.transforms.core :as transforms]
[metabase.util.log :as log]
[methodical.core :as methodical]
- [toucan2.core :as t2])
- (:import
- (java.util Map Set)
- (java.util.concurrent ConcurrentHashMap)))
+ [toucan2.core :as t2]))
(set! *warn-on-reflection* true)
-(defn- current-millis
- "Returns the current epoch millis. Uses java-time so that clock settings can be used in tests."
- []
- (t/to-millis-from-epoch (t/instant)))
-
-(def ^:private entities
- "The list of models to backfill.
+(def ^:private entity-types
+ "The list of entity types to backfill.
- This is not the same as deps.dependency-types/models, because tables shouldn't be backfilled. Instead, links
+ This is not the same as deps.dependency-types/dependency-types, because tables shouldn't be backfilled. Instead, links
involving tables are found via analysis of the other side of the relation."
- [:model/Card
- :model/Transform
- :model/NativeQuerySnippet
- :model/Dashboard
- :model/Document
- :model/Sandbox
- :model/Segment
- :model/Measure])
-
-;; In-memory state for tracking failed entities
-;; Stores {:model/Type {id {:fail-count N :next-retry-timestamp M}}}
-(def ^:private retry-state
- (zipmap entities (repeatedly #(ConcurrentHashMap.))))
-
-;; Stores {:model/Type #{id1 id2 ...}} for entities that have exceeded MAX_RETRIES
-(def ^:private terminally-broken
- (zipmap entities (repeatedly #(ConcurrentHashMap/newKeySet))))
+ [:card :transform :snippet :dashboard :document :sandbox :segment :measure])
(def ^:private max-retries 5)
-(defn- processable-ids [model-kw batch-size]
- (let [target-version models.dependency/current-dependency-analysis-version
- terminally-broken-ids ^Set (terminally-broken model-kw)
- retry-state-map ^Map (retry-state model-kw)
- current-time (current-millis)]
- (into []
- (comp
- (map :id)
- (filter (fn [id]
- (and (not (.contains terminally-broken-ids id))
- (let [entity-retry-info (.get retry-state-map id)]
- (or (nil? entity-retry-info)
- (>= current-time (:next-retry-timestamp entity-retry-info)))))))
- (take batch-size))
- (t2/reducible-select [model-kw :id] :dependency_analysis_version [:< target-version]))))
-
-(def ^:private custom-backfill-events
- {:model/Card :event/card-dependency-backfill
- :model/Dashboard :event/dashboard-dependency-backfill})
-
-(defn- custom-backfill-entity!
- [model-kw event id target-version]
- ;; We don't want to change the entity at all, we just want to update the dependency data and
- ;; mark the entity as processed for this dependency analysis version.
- (let [update-count (t2/update! model-kw id :dependency_analysis_version [:< target-version]
- {:dependency_analysis_version target-version})]
- (when-let [entity (and (pos? update-count)
- (t2/select-one model-kw id))]
- (events/publish-event! event {:object entity}))
- update-count))
-
-(defn- backfill-entity!
- [model-kw id target-version]
- (log/debug "Backfilling " (name model-kw) id)
- (u/prog1
- (t2/with-transaction [_]
- (if-let [event (custom-backfill-events model-kw)]
- (custom-backfill-entity! model-kw event id target-version)
- (t2/update! model-kw id :dependency_analysis_version [:< target-version]
- {:dependency_analysis_version target-version})))
- (log/debug "Backfilled " (name model-kw) id)))
+;;; ------------------------------ Post-deps cleanup multimethod ------------------------------
+
+(defmulti post-deps-cleanup!
+ "Perform entity-specific cleanup after dependencies have been replaced.
+ For example, transforms need to clean up outdated downstream table->transform dependencies.
+ Default is no-op."
+ {:arglists '([entity-type entity])}
+ (fn [entity-type _entity] entity-type))
+
+(defmethod post-deps-cleanup! :default [_ _entity] nil)
+
+(defmethod post-deps-cleanup! :transform [_ {:keys [id target] :as transform}]
+ (let [db-id (transforms/transform-source-database transform)
+ downstream-table-ids (t2/select-fn-set :from_entity_id :model/Dependency
+ :from_entity_type :table
+ :to_entity_type :transform
+ :to_entity_id id)
+ downstream-tables (when (seq downstream-table-ids)
+ (t2/select :model/Table :id [:in downstream-table-ids]))
+ outdated-tables (remove (fn [table]
+ (and (= (:schema table) (:schema target))
+ (= (:name table) (:name target))
+ (or (not db-id)
+ (= db-id (:db_id table)))))
+ downstream-tables)
+ not-found-table-ids (remove (into #{} (map :id) downstream-tables)
+ downstream-table-ids)]
+ (when-let [outdated-downstream-table-ids (seq (into (set not-found-table-ids)
+ (map :id) outdated-tables))]
+ (t2/delete! :model/Dependency
+ :from_entity_type :table
+ :from_entity_id [:in outdated-downstream-table-ids]
+ :to_entity_type :transform
+ :to_entity_id id))))
+
+;;; ------------------------------ Backfill orchestration ------------------------------
+
+(defn- processable-instances [entity-type batch-size]
+ (deps.dependency-status/hydrate-for-deps
+ entity-type
+ (deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))
+
+(defn- compute-deps-for-entity!
+ "Compute and store dependencies for an entity, then update its dependency_status.
+ Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."
+ [entity-type entity]
+ (log/debug "Computing dependencies for" (name entity-type) (:id entity))
+ (t2/with-transaction [_]
+ (let [deps (deps.calculation/calculate-deps entity-type entity)]
+ (models.dependency/replace-dependencies! entity-type (:id entity) deps)
+ (post-deps-cleanup! entity-type entity)
+ (deps.dependency-status/upsert-status! entity-type (:id entity)))))
(defn- backfill-entity-batch!
- [model-kw batch-size]
- (let [model-name (name model-kw)
- target-version models.dependency/current-dependency-analysis-version
- retry-state-map ^Map (retry-state model-kw)
- terminally-broken-set ^Set (terminally-broken model-kw)
- ids (processable-ids model-kw batch-size)] ; Use the new get-processable-ids
- (when (seq ids)
- (log/infof "Processing a batch of %s %s(s)..." (count ids) model-name))
- (reduce (fn [total id]
+ [entity-type batch-size]
+ (let [type-name (name entity-type)
+ instances (processable-instances entity-type batch-size)]
+ (when (seq instances)
+ (log/infof "Processing a batch of %s %s(s)..." (count instances) type-name))
+ (reduce (fn [total entity]
(+ total
(try
- ;; this should update the dependency table via a toucan2 update hook
- (let [update-count (backfill-entity! model-kw id target-version)]
- (.remove retry-state-map id)
- update-count)
+ (compute-deps-for-entity! entity-type entity)
+ 1
(catch Exception e
- (let [current-time (current-millis)
- entity-retry-info (.get retry-state-map id)
- failure-count (inc (:fail-count entity-retry-info 0))
- retry-minutes (* failure-count (deps.settings/dependency-backfill-delay-minutes))
- new-next-retry-timestamp (+ current-time (* retry-minutes 60 1000))]
- (if (> failure-count max-retries)
- (do (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."
- model-name id failure-count)
- (.add terminally-broken-set id)
- (.remove retry-state-map id)) ; Remove from retry map
- (do (log/warnf e "Entity %s %s failed, failure count: %d, next retry no sooner than %d minutes."
- model-name id failure-count retry-minutes)
- (.put retry-state-map id {:fail-count failure-count
- :next-retry-timestamp new-next-retry-timestamp}))))
+ (let [id (:id entity)]
+ (try
+ (deps.dependency-status/record-failure!
+ entity-type id max-retries
+ (deps.settings/dependency-backfill-delay-minutes))
+ (let [{:keys [fail_count terminal]} (t2/select-one :model/DependencyStatus
+ :entity_type entity-type
+ :entity_id id)]
+ (if terminal
+ (log/errorf e "Entity %s %s failed %d times, marking as terminally broken."
+ type-name id fail_count)
+ (log/warnf e "Entity %s %s failed, failure count: %d."
+ type-name id fail_count)))
+ (catch Exception record-ex
+ (log/errorf e "Entity %s %s failed during dependency calculation."
+ type-name id)
+ (log/errorf record-ex "Additionally, failed to record the failure for %s %s."
+ type-name id))))
0))))
0
- ids)))
+ instances)))
(defn- backfill-dependencies!
"Job to backfill dependencies for all entities.
Returns true if a full batch has been selected, nil or false otherwise."
[]
(when (premium-features/has-feature? :dependencies)
- (-> (reduce (fn [batch-size model-kw]
+ (-> (reduce (fn [batch-size entity-type]
(if (< batch-size 1)
(reduced 0)
- (let [processed (backfill-entity-batch! model-kw batch-size)]
+ (let [processed (backfill-entity-batch! entity-type batch-size)]
(when (pos? processed)
(log/info "Updated" processed "entities."))
(- batch-size processed))))
(deps.settings/dependency-backfill-batch-size)
- entities)
+ entity-types)
(< 1))))
(defn- has-pending-retries? []
- (some (fn [^Map model-retry-state]
- (not (.isEmpty model-retry-state)))
- (vals retry-state)))
+ (deps.dependency-status/has-pending-retries?))
(declare schedule-next-run!)
@@ -186,12 +168,17 @@
:delay-in-seconds delay-in-seconds
:scheduler scheduler})))
+(defn trigger-backfill-job!
+ "Trigger the BackfillDependencies job to run after a brief delay.
+ The 1-second delay ensures the calling transaction has committed before
+ the job checks for stale entities."
+ []
+ (schedule-next-run! 1))
+
(defmethod task/init! ::DependencyBackfill [_]
(if (pos? (deps.settings/dependency-backfill-batch-size))
- (schedule-next-run! (if config/is-test?
- 0
- (deps.task-util/job-initial-delay
- (deps.settings/dependency-backfill-variance-minutes))))
+ (schedule-next-run! (deps.task-util/job-initial-delay
+ (deps.settings/dependency-backfill-variance-minutes)))
(log/info "Not starting dependency backfill job because the batch size is not positive")))
(derive ::backfill :metabase/event)
@@ -201,4 +188,4 @@
(methodical/defmethod events/publish-event! ::backfill
[_ _]
(when (premium-features/has-feature? :dependencies)
- (backfill-dependencies!)))
+ (trigger-backfill-job!)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
index 1f7648ddc61..c308ecda559 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
@@ -26,14 +26,15 @@
(defn- check-entities!
"Process entities, draining all stale entities before returning.
- Returns true if the full batch was used (more work may remain)."
+ Continues looping as long as there are stale entities — this is important because
+ [[deps.findings/analyze-and-propagate!]] marks immediate dependents stale during processing,
+ which need to be picked up in subsequent waves."
[]
(when (premium-features/has-feature? :dependencies)
(loop []
- (let [used-full-batch? (process-one-batch!)]
- (if (and used-full-batch? (deps.analysis-finding/has-stale-entities?))
- (recur)
- used-full-batch?)))))
+ (process-one-batch!)
+ (when (deps.analysis-finding/has-stale-entities?)
+ (recur)))))
(declare schedule-next-run!)
@@ -70,7 +71,8 @@
(log/info "Not starting dependency entity check job because the batch size is not positive")))
(defn trigger-entity-check-job!
- "Trigger the DependencyEntityCheck job to run immediately.
- Use this when entities have been marked stale and need re-analysis."
+ "Trigger the DependencyEntityCheck job to run after a brief delay.
+ The 1-second delay ensures the calling transaction has committed before
+ the job checks for stale entities."
[]
- (schedule-next-run! 0))
+ (schedule-next-run! 1))
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index b8b4ebee670..56abd204bef 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -79,6 +79,7 @@
"DataPermissions"
"DatabaseRouter"
"Dependency"
+ "DependencyStatus"
"DocumentBookmark"
"HTTPAction"
"ImplicitAction"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index f466f68ce61..228a50695f1 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -7,7 +7,7 @@
[metabase-enterprise.dependencies.core :as dependencies]
[metabase-enterprise.dependencies.events]
[metabase-enterprise.dependencies.findings :as dependencies.findings]
- [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.collections.models.collection :as collection]
[metabase.core.core :as mbc]
[metabase.events.core :as events]
@@ -148,8 +148,9 @@
(testing "POST /api/ee/dependencies/check-card"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [card (card/create-card! (basic-card) user)
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card"
(assoc (card/create-card! (basic-card "Product question" :products)
user)
@@ -165,7 +166,7 @@
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/Dashboard dashboard {}
:model/Document document {}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [metadata-provider (mt/metadata-provider)
;; Create base card querying real orders table
base-card (card/create-card! (basic-card) user)
@@ -187,6 +188,7 @@
:type :question
:dataset_query proposed-query
:result_metadata nil}]
+ (deps.test/synchronously-run-backfill!)
(is (=? {:success false
:bad_cards [{:id (:id dashboard-card)
:dashboard_id (:id dashboard)
@@ -202,7 +204,7 @@
(mt/dataset test-data
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [mp (mt/metadata-provider)
;; Create base card querying real orders table
base-card (card/create-card! (basic-card) user)
@@ -221,6 +223,7 @@
:type :question
:dataset_query proposed-query
:result_metadata nil}
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card" proposed-card)]
(is (=? {:success false
:bad_cards [{:id (:id dependent-card)}]
@@ -232,7 +235,7 @@
(mt/dataset test-data
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [mp (mt/metadata-provider)
orders-query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
base-query (-> orders-query
@@ -258,6 +261,7 @@
:type :question
:dataset_query proposed-query
:result_metadata nil}
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card"
proposed-card)]
(is (=? {:success false
@@ -270,7 +274,7 @@
(mt/dataset test-data
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [mp (mt/metadata-provider)
;; Create base card querying real orders table
base-card (card/create-card! (basic-card) user)
@@ -297,6 +301,7 @@
:type :question
:dataset_query proposed-query
:result_metadata nil}
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card" proposed-card)]
(is (=? {:success false
:bad_cards #{(:id dependent-card-1) (:id dependent-card-2)}
@@ -308,7 +313,7 @@
(mt/dataset test-data
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [mp (mt/metadata-provider)
;; Create base card querying real orders table
base-card (card/create-card! (basic-card) user)
@@ -325,6 +330,7 @@
:type :question
:dataset_query proposed-query
:result_metadata nil}
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-card" proposed-card)]
(is (=? {:success false
:bad_cards [{:id (:id dependent-card)}]
@@ -355,7 +361,7 @@
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/NativeQuerySnippet {snippet-id :id snippet-name :name} {:name "filter-snippet"
:content "WHERE SUBTOTAL > 100"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [tag-name (str "snippet: " snippet-name)
mp (mt/metadata-provider)
native-query (-> (lib/native-query mp (format "SELECT * FROM ORDERS %s" (str "{{" tag-name "}}")))
@@ -369,6 +375,7 @@
:display :table
:visualization_settings {}}
user)
+ _ (deps.test/synchronously-run-backfill!)
proposed-content "WHERE NONEXISTENT_COLUMN > 100"
response (mt/user-http-request :rasta :post 200 "ee/dependencies/check-snippet"
{:id snippet-id
@@ -381,10 +388,11 @@
(deftest graph-test
(testing "GET /api/ee/dependencies/graph"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user)
{card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph" :id card-id-2 :type "card")
creator {:email "[email protected]"
:id (:id user)}]
@@ -441,10 +449,11 @@
(testing "GET /api/ee/dependencies/graph with table as root node"
(mt/dataset test-data
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [_card-1 (card/create-card! (basic-card "Card 1" :orders) user)
_card-2 (card/create-card! (basic-card "Card 2" :orders) user)
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
:id (mt/id :orders)
:type "table")]
@@ -459,10 +468,11 @@
(deftest dependents-test
(testing "GET /api/ee/dependencies/graph/dependents"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card) user)
{card-id-2 :id} (card/create-card! (wrap-card dependency-card) user)
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id card-id-1
:type "card"
@@ -490,7 +500,7 @@
(deftest ^:sequential dependents-multiple-types-test
(testing "GET /api/ee/dependencies/graph/dependents with multiple dependent-types"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/DashboardCard]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/DashboardCard]
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/Dashboard {dashboard-id :id} {:name "Test Dashboard"}]
(let [{card-id-1 :id :as dependency-card} (card/create-card! (basic-card "Base card - multi") user)
@@ -501,7 +511,8 @@
:col 0
:size_x 4
:size_y 4})
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(testing "single dependent-types value (backward compatibility)"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id card-id-1
@@ -528,11 +539,12 @@
(deftest ^:sequential dependents-multiple-card-types-test
(testing "GET /api/ee/dependencies/graph/dependents with multiple dependent-card-types"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [{dependency-card-id :id :as dependency-card} (card/create-card! (basic-card "Base card - cardtypes") user)
_question-card (card/create-card! (assoc (wrap-card dependency-card) :name "Question card") user)
_model-card (card/create-card! (assoc (wrap-card dependency-card) :name "Model card" :type :model) user)]
+ (deps.test/synchronously-run-backfill!)
(testing "single dependent-card-types value"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id dependency-card-id
@@ -568,12 +580,13 @@
(deftest graph-archived-card-test
(testing "GET /api/ee/dependencies/graph with archived parameter"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [base-card (card/create-card! (basic-card "Archived Base Card") user)
dependent-card (card/create-card! (wrap-card base-card) user)]
(card/update-card! {:card-before-update base-card
:card-updates {:archived true}})
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived card from dependencies"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
:id (:id dependent-card)
@@ -594,12 +607,13 @@
(deftest dependents-archived-card-test
(testing "GET /api/ee/dependencies/graph/dependents with archived parameter"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}]
(let [base-card (card/create-card! (basic-card "Base Card") user)
dependent-card (card/create-card! (wrap-card base-card) user)]
(card/update-card! {:card-before-update dependent-card
:card-updates {:archived true}})
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived dependent"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
:id (:id base-card)
@@ -620,7 +634,7 @@
(testing "GET /api/ee/dependencies/graph/dependents?broken=true - only returns entities that are broken"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card dependent-card] (lib-be/with-metadata-provider-cache
(let [model-card (create-model-card! user "Model Card - brokentest")
@@ -630,7 +644,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card)
(let [next-card (create-dependent-card-on-model! user model-card "Another Dependent Card - brokentest" {:table :products})]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id next-card))
(run-analysis-for-card! (:id dependent-card))
(let [response2 (mt/user-http-request :crowberto :get 200 (str "ee/dependencies/graph/dependents?broken=true&type=card&id=" (:id model-card)))]
@@ -646,8 +660,9 @@
(mt/with-non-admin-groups-no-root-collection-perms
(mt/with-temp [:model/Collection collection {}
:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [card (card/create-card! (assoc (basic-card) :collection_id (u/the-id collection)) user)]
+ (deps.test/synchronously-run-backfill!)
(testing "Returns 403 when user lacks read permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :post 403 "ee/dependencies/check-card"
@@ -701,11 +716,12 @@
(mt/with-premium-features #{:dependencies :transforms-basic}
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/Transform transform {}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [base-card (card/create-card! (basic-card) user)
proposed-card {:id (:id base-card)
:type :question
:dataset_query (:dataset_query base-card)}]
+ (deps.test/synchronously-run-backfill!)
;; Mock errors-from-proposed-edits to report the transform as broken
(with-redefs [dependencies/errors-from-proposed-edits
(constantly {:transform {(:id transform) #{:some-error}}})]
@@ -726,8 +742,9 @@
(mt/with-non-admin-groups-no-root-collection-perms
(mt/with-temp [:model/Collection collection {}
:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [card (card/create-card! (assoc (basic-card) :collection_id (u/the-id collection)) user)]
+ (deps.test/synchronously-run-backfill!)
(testing "Returns 403 when user lacks read permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/dependencies/graph"
@@ -745,8 +762,9 @@
(mt/with-non-admin-groups-no-root-collection-perms
(mt/with-temp [:model/Collection collection {}
:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [{card-id :id} (card/create-card! (assoc (basic-card) :collection_id (:id collection)) user)]
+ (deps.test/synchronously-run-backfill!)
(testing "Returns 403 when user lacks read permissions"
(is (= "You don't have permissions to do that."
(mt/user-http-request :rasta :get 403 "ee/dependencies/graph/dependents"
@@ -770,7 +788,7 @@
(mt/with-temp [:model/Collection readable-collection {}
:model/Collection unreadable-collection {}
:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [readable-base (card/create-card! (assoc (basic-card "Readable")
:collection_id (:id readable-collection)) user)
unreadable-base (card/create-card! (assoc (basic-card "Unreadable")
@@ -778,6 +796,7 @@
top-card (card/create-card! (assoc (wrap-two-cards readable-base unreadable-base)
:collection_id (:id readable-collection))
user)]
+ (deps.test/synchronously-run-backfill!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees complete upstream graph through readable path"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -810,7 +829,7 @@
(mt/with-temp [:model/Collection readable-collection {}
:model/Collection unreadable-collection {}
:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [base-card (card/create-card! (assoc (basic-card) :collection_id (:id readable-collection)) user)
readable-dependent (card/create-card! (assoc (wrap-card base-card)
:collection_id (:id readable-collection))
@@ -818,6 +837,7 @@
unreadable-dependent (card/create-card! (assoc (wrap-card base-card)
:collection_id (:id unreadable-collection))
user)]
+ (deps.test/synchronously-run-backfill!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only readable dependents"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
@@ -838,7 +858,7 @@
(mt/with-temp [:model/Collection readable-collection {}
:model/Collection unreadable-collection {}
:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [base-card (card/create-card! (assoc (basic-card) :collection_id (:id readable-collection)) user)
unreadable-middle (card/create-card! (assoc (wrap-card base-card)
:collection_id (:id unreadable-collection))
@@ -849,6 +869,7 @@
end-card (card/create-card! (assoc (wrap-two-cards unreadable-middle readable-alternate)
:collection_id (:id readable-collection))
user)]
+ (deps.test/synchronously-run-backfill!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "Diamond pattern: complete upstream graph via readable path"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -887,12 +908,13 @@
(mt/with-temp [:model/Collection readable-collection {}
:model/Collection unreadable-collection {}
:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [unreadable-base (card/create-card! (assoc (basic-card "Unreadable")
:collection_id (:id unreadable-collection)) user)
top-card (card/create-card! (assoc (wrap-card unreadable-base)
:collection_id (:id readable-collection))
user)]
+ (deps.test/synchronously-run-backfill!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only the top card in the graph"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -918,7 +940,7 @@
:model/User user {:email "[email protected]"}
:model/NativeQuerySnippet {snippet-id :id snippet-name :name} {:name "test-snippet"
:content "WHERE ID > 10"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [tag-name (str "snippet: " snippet-name)
mp (mt/metadata-provider)
native-query (fn []
@@ -940,6 +962,7 @@
:visualization_settings {}
:collection_id (:id unreadable-collection)}
user)]
+ (deps.test/synchronously-run-backfill!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only readable cards as dependents of the snippet"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph/dependents"
@@ -961,12 +984,13 @@
(mt/with-temp [:model/Collection readable-collection {}
:model/User user {:email "[email protected]"}]
(mt/with-temp-copy-of-db
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-no-data-perms-for-all-users!
(perms/set-table-permission! (perms/all-users-group) (mt/id :orders) :perms/view-data :blocked)
(perms/set-table-permission! (perms/all-users-group) (mt/id :orders) :perms/create-queries :no)
(let [card (card/create-card! (assoc (basic-card "Card on orders")
:collection_id (:id readable-collection)) user)]
+ (deps.test/synchronously-run-backfill!)
(perms/grant-collection-read-permissions! (perms/all-users-group) readable-collection)
(testing "User sees only the card, table is filtered out"
(let [response (mt/user-http-request :rasta :get 200 "ee/dependencies/graph"
@@ -986,13 +1010,14 @@
(testing "Graph endpoints return dashboard data for cards in dashboards"
(mt/with-premium-features #{:dependencies}
(mt/with-current-user (mt/user->id :rasta)
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/Dashboard dashboard {:name "Test Dashboard"}]
(let [base-card (card/create-card! (basic-card "Base Card") user)
dashboard-card (card/create-card! (assoc (wrap-card base-card)
:dashboard_id (:id dashboard))
user)]
+ (deps.test/synchronously-run-backfill!)
(testing "GET /api/ee/dependencies/graph returns dashboard with :id and :name"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id (:id dashboard-card)
@@ -1018,13 +1043,14 @@
(deftest graph-returns-document-for-cards-test
(testing "Graph endpoints return document data for cards in documents"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/Document document {:name "Test Document"}]
(let [base-card (card/create-card! (basic-card "Base Card") user)
document-card (card/create-card! (assoc (wrap-card base-card)
:document_id (:id document))
user)]
+ (deps.test/synchronously-run-backfill!)
(testing "GET /api/ee/dependencies/graph returns document with :id and :name"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id (:id document-card)
@@ -1057,6 +1083,7 @@
:table_id products-id
:definition {:filter [:> [:field price-field-id nil] 50]}}]
(events/publish-event! :event/segment-create {:object segment :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id segment-id
:type "segment")
@@ -1071,7 +1098,7 @@
(deftest graph-segment-dependents-test
(testing "GET /api/ee/dependencies/graph/dependents with segment"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)]
(mt/with-temp [:model/Segment {segment-id :id :as segment} {:name "High Value Products"
@@ -1085,6 +1112,7 @@
query-with-segment (-> (lib/query mp products)
(lib/filter (lib.metadata/segment mp segment-id)))
card (card/create-card! (card-with-query "Card using segment" query-with-segment) user)
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"
:id segment-id
:type "segment"
@@ -1107,6 +1135,7 @@
:definition (-> (lib/query mp products)
(lib/aggregate (lib/sum price)))}]
(events/publish-event! :event/measure-create {:object measure :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id measure-id
:type "measure")
@@ -1132,6 +1161,7 @@
:definition (-> (lib/query mp products)
(lib/aggregate (lib/sum price)))}]
(events/publish-event! :event/measure-create {:object measure-a :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [mp' (mt/metadata-provider)]
(mt/with-temp [:model/Measure {measure-b-id :id :as measure-b} {:name "Measure B"
:table_id products-id
@@ -1139,6 +1169,7 @@
(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id)
(lib/sum rating))))}]
(events/publish-event! :event/measure-create {:object measure-b :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id measure-b-id
:type "measure")
@@ -1158,7 +1189,7 @@
(deftest graph-measure-dependents-test
(testing "GET /api/ee/dependencies/graph/dependents with measure"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [mp (mt/metadata-provider)
products-id (mt/id :products)
products (lib.metadata/table mp products-id)
@@ -1174,6 +1205,7 @@
query-with-measure (-> (lib/query mp' products)
(lib/aggregate (lib.metadata/measure mp' measure-id)))
card (card/create-card! (card-with-query "Card using measure" query-with-measure) user)
+ _ (deps.test/synchronously-run-backfill!)
response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"
:id measure-id
:type "measure"
@@ -1234,7 +1266,7 @@
;; Archive measure B (the middle of the chain)
(mt/user-http-request :crowberto :put 200 (str "measure/" measure-b-id)
{:archived true :revision_message "Archive middle measure"})
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "after archiving measure B, it and measure A are excluded (chain broken)"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph"
:id measure-c-id
@@ -1304,7 +1336,9 @@
:dataset_query (->> referenced-card-id
(lib.metadata/card mp)
(lib/query mp))}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id referenced-card-id) :user-id (mt/user->id :crowberto)})
+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id unreffed-card-id) :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=unreftest")]
(is (=? {:data [{:id unreffed-card-id
:type "card"
@@ -1317,10 +1351,11 @@
(let [mp (mt/metadata-provider)]
(mt/with-temp [:model/Table {unreffed-table-id :id} {:name "Unreferenced Table - unreftest"}
:model/Table {referenced-table-id :id} {:name "Referenced Table - unreftest"}
- :model/Card _card {:name "Referencing Card"
- :type :question
- :dataset_query (lib/query mp (lib.metadata/table mp referenced-table-id))}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ :model/Card {card-id :id} {:name "Referencing Card"
+ :type :question
+ :dataset_query (lib/query mp (lib.metadata/table mp referenced-table-id))}]
+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=unreftest")]
(is (=? {:data [{:id unreffed-table-id
:type "table"
@@ -1350,7 +1385,7 @@
:output-schema "PUBLIC"
:output-table "referenced_transform_table"
:transform-id referenced-transform-id}})
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(try
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]
(is (=? {:data [{:id unreffed-transform-id
@@ -1376,10 +1411,11 @@
:type :snippet
:snippet-name snippet-name
:snippet-id referenced-snippet-id}}))]
- (mt/with-temp [:model/Card _card {:name "Card using snippet"
- :type :question
- :dataset_query native-query}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (mt/with-temp [:model/Card {card-id :id} {:name "Card using snippet"
+ :type :question
+ :dataset_query native-query}]
+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id card-id) :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=unreftest")]
(is (=? {:data [{:id unreffed-snippet-id
:type "snippet"
@@ -1391,15 +1427,15 @@
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - unreftest"}
:model/Dashboard {referenced-dashboard-id :id} {:name "Referenced Dashboard - unreftest"}
- :model/Document _ {:name "Document with dashboard link"
- :dependency_analysis_version 0
- :document {:type "doc"
- :content [{:type "paragraph"
- :content [{:type "smartLink"
- :attrs {:entityId referenced-dashboard-id
- :model "dashboard"}}]}]}
- :content_type "application/json+vnd.prose-mirror"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ :model/Document {document-id :id} {:name "Document with dashboard link"
+ :document {:type "doc"
+ :content [{:type "paragraph"
+ :content [{:type "smartLink"
+ :attrs {:entityId referenced-dashboard-id
+ :model "dashboard"}}]}]}
+ :content_type "application/json+vnd.prose-mirror"}]
+ (events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id document-id) :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=unreftest")]
(is (=? {:data [{:id unreffed-dashboard-id
:type "dashboard"
@@ -1417,7 +1453,8 @@
:attrs {:entityId referenced-document-id
:model "document"}}]}]}
:content_type "application/json+vnd.prose-mirror"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (events/publish-event! :event/document-create {:object (t2/select-one :model/Document :id unreffed-document-id) :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=unreftest")]
(is (=? {:data [{:id unreffed-document-id
:type "document"
@@ -1436,7 +1473,7 @@
:model/Sandbox {sandbox-id :id} {:group_id group-id
:table_id (mt/id :products)
:card_id sandbox-card-id}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=sandbox")]
(is (=? {:data [{:id sandbox-id
:type "sandbox"
@@ -1454,7 +1491,7 @@
:model/Card {unreffed-metric-id :id} {:name "B - Unreferenced Metric - cardtype"
:type :metric
:dataset_query (lib/query mp products)}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "filtering by model only"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=model&query=cardtype")]
(is (=? {:data [{:id unreffed-model-id
@@ -1493,7 +1530,7 @@
:type :question
:archived true
:dataset_query (lib/query mp products)}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived card"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=archivedtest")
card-ids (set (map :id (:data response)))]
@@ -1511,7 +1548,7 @@
(mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - archivedtest"}
:model/Dashboard {archived-dashboard-id :id} {:name "Archived Unreferenced Dashboard - archivedtest"
:archived true}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived dashboard"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest")
dashboard-ids (set (map :id (:data response)))]
@@ -1529,7 +1566,7 @@
(mt/with-temp [:model/Document {unreffed-document-id :id} {:name "Unreferenced Document - archivedtest"}
:model/Document {archived-document-id :id} {:name "Archived Unreferenced Document - archivedtest"
:archived true}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived document"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest")
document-ids (set (map :id (:data response)))]
@@ -1549,7 +1586,7 @@
:model/NativeQuerySnippet {archived-snippet-id :id} {:name "Archived Unreferenced Snippet - archivedtest"
:content "WHERE ID > 20"
:archived true}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived snippet"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest")
snippet-ids (set (map :id (:data response)))]
@@ -1575,7 +1612,7 @@
:archived true}]
(events/publish-event! :event/segment-create {:object unreffed-segment :user-id (mt/user->id :crowberto)})
(events/publish-event! :event/segment-create {:object archived-segment :user-id (mt/user->id :crowberto)})
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived segment"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest")
segment-ids (set (map :id (:data response)))]
@@ -1600,7 +1637,7 @@
:db_id (mt/id)
:active true
:visibility_type "hidden"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes inactive and hidden tables"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest")
table-ids (set (map :id (:data response)))]
@@ -1620,7 +1657,7 @@
(mt/with-temp [:model/Table {table1-id :id} {:name "Table 1 - unreftest"}
:model/Table {table2-id :id} {:name "Table 2 - unreftest"}
:model/Table {table3-id :id} {:name "Table 3 - unreftest"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(is (=? {:data [{:id table1-id} {:id table2-id}]
:total 3
:offset 0
@@ -1669,7 +1706,7 @@
:archived true}]
(events/publish-event! :event/measure-create {:object unreffed-measure :user-id (mt/user->id :crowberto)})
(events/publish-event! :event/measure-create {:object archived-measure :user-id (mt/user->id :crowberto)})
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "archived=false (default) excludes archived measure"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest")
measure-ids (set (map :id (:data response)))]
@@ -1703,7 +1740,7 @@
:model/Card {card-regular :id} {:name "Card Regular - personalcolltest"
:type :question
:dataset_query (lib/query mp products)}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "include-personal-collections=false (default) excludes cards in personal collections"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card-types=question&query=personalcolltest")
card-ids (set (map :id (:data response)))]
@@ -1727,7 +1764,7 @@
:model/Dashboard {dash-in-personal :id} {:name "Dashboard in Personal - personalcolltest"
:collection_id personal-coll-id}
:model/Dashboard {dash-regular :id} {:name "Dashboard Regular - personalcolltest"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(testing "include-personal-collections=false (default) excludes dashboards in personal collections"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=personalcolltest")
dashboard-ids (set (map :id (:data response)))]
@@ -1751,7 +1788,7 @@
:model/Document _ {:name "F Document sorttest"}
:model/Segment _ {:name "G Segment sorttest"}
:model/Measure _ {:name "H Measure sorttest"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(doseq [sort-direction [:asc :desc]]
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
@@ -1816,7 +1853,7 @@
:table_id table1-id}
:model/Measure _ {:name "Measure with Table 2 sorttest"
:table_id table2-id}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(doseq [sort-direction [:asc :desc]]
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
@@ -1846,7 +1883,7 @@
:collection_id collection1-id}
:model/NativeQuerySnippet _ {:name "SQL snippets sorttest"}
:model/Transform _ {:name "Transforms sorttest"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(doseq [sort-direction [:asc :desc]]
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
@@ -1865,7 +1902,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - returns entities that are SOURCE of downstream errors"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card dependent-card]
(lib-be/with-metadata-provider-cache
@@ -1876,7 +1913,7 @@
(break-model-card! model-card))
;; Run analysis in a fresh metadata provider cache session to detect the broken reference
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card)))
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=brokentest")]
(is (= [(:id model-card)] (mapv :id (:data response)))
@@ -1886,7 +1923,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - types parameter filters results"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card-1 model-card-2 dependent-card-1 dependent-card-2]
(lib-be/with-metadata-provider-cache
@@ -1901,7 +1938,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(testing "filtering by card returns only card sources"
@@ -1918,7 +1955,7 @@
(testing "GET /api/ee/dependencies/graph/breaking with archived parameter for source cards"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[active-model archived-model dependent-card-1 dependent-card-2]
(lib-be/with-metadata-provider-cache
@@ -1936,7 +1973,7 @@
(break-model-card! (t2/select-one :model/Card :id (:id archived-model))))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(testing "archived=false (default) excludes archived source card"
@@ -1954,7 +1991,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - model breaking multiple dependents appears once"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card dependent-card-1 dependent-card-2]
(lib-be/with-metadata-provider-cache
@@ -1967,7 +2004,7 @@
(break-model-card! model-card))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/breaking?types=card&query=multipledependents")
@@ -1982,7 +2019,7 @@
:model/User creator {:email "[email protected]"}
:model/Collection {personal-coll-id :id} {:personal_owner_id user-id
:name "Test Personal Collection"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-in-personal model-regular dependent-card-1 dependent-card-2]
(lib-be/with-metadata-provider-cache
@@ -1998,7 +2035,7 @@
(break-model-card! model-regular))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(testing "include-personal-collections=false (default) excludes source cards in personal collections"
@@ -2016,7 +2053,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - should paginate results"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card-1 model-card-2 dependent-card-1 dependent-card-2]
(lib-be/with-metadata-provider-cache
@@ -2031,7 +2068,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(is (=? {:data [{:id (:id model-card-1)}]
@@ -2049,7 +2086,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - sorting by name"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card-a model-card-b dependent-card-a dependent-card-b]
(lib-be/with-metadata-provider-cache
@@ -2064,7 +2101,7 @@
(break-model-card! model-card-b))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-a))
(run-analysis-for-card! (:id dependent-card-b)))
(doseq [sort-direction [:asc :desc]]
@@ -2086,7 +2123,7 @@
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/Collection {collection1-id :id} {:name "B Collection"}
:model/Collection {collection2-id :id} {:name "A Collection"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-in-coll1 model-in-coll2 dependent-card-1 dependent-card-2]
(lib-be/with-metadata-provider-cache
@@ -2101,7 +2138,7 @@
(break-model-card! model-in-coll2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(doseq [sort-direction [:asc :desc]]
@@ -2121,7 +2158,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - sorting by dependents with errors count"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card-1 model-card-2 dependent-card-1a dependent-card-1b dependent-card-2a]
(lib-be/with-metadata-provider-cache
@@ -2138,7 +2175,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1a))
(run-analysis-for-card! (:id dependent-card-1b))
(run-analysis-for-card! (:id dependent-card-2a)))
@@ -2159,7 +2196,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - sorting by dependents with errors count"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
;; Model 1 breaks 2 dependent cards (2 unique analyzed entities with errors)
;; Model 2 breaks 1 dependent card (1 unique analyzed entity with errors)
@@ -2177,7 +2214,7 @@
(break-model-card! model-card-2))
;; Run analysis in a fresh metadata provider cache session to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1a))
(run-analysis-for-card! (:id dependent-card-1b))
(run-analysis-for-card! (:id dependent-card-2a)))
@@ -2210,7 +2247,7 @@
(defmacro ^:private with-dependents-test!
[[user-binding base-card-binding] & body]
`(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user# {:email "[email protected]"}]
(let [~user-binding user#
~base-card-binding (card/create-card! (basic-card "Base") user#)]
@@ -2222,6 +2259,7 @@
(create-dependent! base-card user "Alpha")
(create-dependent! base-card user "Beta")
(create-dependent! base-card user "Gamma")
+ (deps.test/synchronously-run-backfill!)
(testing "filters by name"
(is (=? [{:data {:name "Alpha"}}]
(get-dependents base-card-id :query "Alpha"))))
@@ -2237,6 +2275,7 @@
(mt/with-temp [:model/Collection {coll-id :id} {:name "SpecialCollection"}]
(create-dependent! base-card user "Card in root")
(create-dependent! base-card user "Card in collection" :collection_id coll-id)
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:data {:name "Card in collection"}}]
(get-dependents base-card-id :query "SpecialCollection")))))))
@@ -2249,6 +2288,7 @@
(create-dependent! base-card user "In Personal" :collection_id personal-coll-id)
(create-dependent! base-card user "In Sub" :collection_id sub-coll-id)
(create-dependent! base-card user "Regular")
+ (deps.test/synchronously-run-backfill!)
(testing "default excludes personal collections"
(is (=? [{:data {:name "Regular"}}]
(get-dependents base-card-id))))
@@ -2261,6 +2301,7 @@
(create-dependent! base-card user "C Card")
(create-dependent! base-card user "A Card")
(create-dependent! base-card user "B Card")
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:data {:name "A Card"}} {:data {:name "B Card"}} {:data {:name "C Card"}}]
(get-dependents base-card-id :sort-column :name :sort-direction :asc)))
(is (=? [{:data {:name "C Card"}} {:data {:name "B Card"}} {:data {:name "A Card"}}]
@@ -2275,6 +2316,7 @@
(create-dependent! base-card user "In C" :collection_id coll-c)
(create-dependent! base-card user "In A" :collection_id coll-a)
(create-dependent! base-card user "In B" :collection_id coll-b)
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:data {:name "In A"}} {:data {:name "In B"}} {:data {:name "In C"}}]
(get-dependents base-card-id :sort-column :location :sort-direction :asc)))
(is (=? [{:data {:name "In C"}} {:data {:name "In B"}} {:data {:name "In A"}}]
@@ -2286,6 +2328,7 @@
(let [{low-id :id} (create-dependent! base-card user "Low")
{mid-id :id} (create-dependent! base-card user "Mid")
{high-id :id} (create-dependent! base-card user "High")]
+ (deps.test/synchronously-run-backfill!)
(t2/update! :model/Card low-id {:view_count 10})
(t2/update! :model/Card mid-id {:view_count 50})
(t2/update! :model/Card high-id {:view_count 100})
@@ -2303,7 +2346,8 @@
{high-id :id} (create-dependent! base-card user "High")]
(t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id base-card-id
:row 0 :col 0 :size_x 4 :size_y 4})
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (events/publish-event! :event/dashboard-update {:object (t2/select-one :model/Dashboard :id dashboard-id) :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(t2/update! :model/Card low-id {:view_count 10})
(t2/update! :model/Card high-id {:view_count 100})
(is (=? [{:data {:view_count 10}} {:data {:view_count 100}} {:data {:view_count 200}}]
@@ -2318,6 +2362,7 @@
(create-dependent! base-card user "A Match")
(create-dependent! base-card user "B Match")
(create-dependent! base-card user "Should not appear")
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:data {:name "A Match"}} {:data {:name "B Match"}} {:data {:name "C Match"}}]
(get-dependents base-card-id :query "Match" :sort-column :name :sort-direction :asc))))))
@@ -2386,7 +2431,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - pagination and sorting work with error visibility filtering"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create cards in one metadata provider cache session
(let [[model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a archived-dep-b]
(lib-be/with-metadata-provider-cache
@@ -2408,7 +2453,7 @@
(break-model-card! model-card-b))
;; Run analysis to detect broken references
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id visible-dep-a))
(run-analysis-for-card! (:id visible-dep-b))
(run-analysis-for-card! (:id archived-dep-a))
@@ -2482,7 +2527,7 @@
(testing "GET /api/ee/dependencies/graph/breaking - sorting counts only visible errors, not archived"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Setup: Model A has fewer VISIBLE errors but more TOTAL errors (due to archived dependents)
;; Model B has more VISIBLE errors but fewer TOTAL errors
;; If sorting counts all errors: A (3 total) > B (2 total) -> B first in asc
@@ -2547,11 +2592,11 @@
(deftest ^:sequential unreferenced-pagination-with-archived-items-test
(testing "GET /api/ee/dependencies/graph/unreferenced - pagination works correctly with archived items"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Dependency]
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/Card _ {:name "A Card - unreftest" :archived true}
:model/Card {card2-id :id} {:name "B Card - unreftest"}
:model/Card {card3-id :id} {:name "C Card - unreftest"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "card"
@@ -2569,18 +2614,20 @@
(deftest ^:sequential unreferenced-pagination-with-archived-dependents-test
(testing "GET /api/ee/dependencies/graph/unreferenced - should return items if all dependents are archived"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Dependency]
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/Card {card1-id :id, :as card1} {:name "A Card - unreftest"}
:model/Card card2 {:name "B Card - unreftest"}
- :model/Card _ {:name "C Card - unreftest"
+ :model/Card {card3-id :id} {:name "C Card - unreftest"
:dataset_query (wrap-card-query card1)
:archived true}
:model/Card {card4-id :id} {:name "D Card - unreftest"
:dataset_query (wrap-card-query card2)}
- :model/Card _ {:name "E Card - unreftest"
+ :model/Card {card5-id :id} {:name "E Card - unreftest"
:dataset_query (wrap-card-query card2)
:archived true}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (doseq [id [card1-id (:id card2) card3-id card4-id card5-id]]
+ (events/publish-event! :event/card-create {:object (t2/select-one :model/Card :id id) :user-id (mt/user->id :crowberto)}))
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "card"
@@ -2621,6 +2668,7 @@
:type :query
:query {:source-table table-1-id}}}]
(events/publish-event! :event/card-create {:object card :user-id (:creator_id card)})
+ (deps.test/synchronously-run-backfill!)
(testing "after creating card that depends on table-1, only table-2 is unused"
(is (= #{table-2-id}
(->> (mt/user-http-request :crowberto :get 200 "table" :unused-only true)
@@ -2635,7 +2683,7 @@
(testing "GET /api/ee/dependencies/graph/broken - returns broken dependents filtered by source"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Setup: two models, each with one dependent
(let [[model-card-1 model-card-2 dependent-card-1 dependent-card-2]
(lib-be/with-metadata-provider-cache
@@ -2650,7 +2698,7 @@
(break-model-card! model-card-1)
(break-model-card! model-card-2))
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-1))
(run-analysis-for-card! (:id dependent-card-2)))
(testing "returns only broken dependents for the specified source (not other sources)"
@@ -2664,7 +2712,7 @@
(testing "GET /api/ee/dependencies/graph/broken - count matches and sorting works"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Setup: one model with two dependents (named for sorting)
(let [[model-card dependent-card-a dependent-card-b]
(lib-be/with-metadata-provider-cache
@@ -2677,7 +2725,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card-a))
(run-analysis-for-card! (:id dependent-card-b)))
(testing "count matches dependents-with-errors from /graph/breaking"
@@ -2721,7 +2769,7 @@
(mt/with-non-admin-groups-no-root-collection-perms
(mt/with-temp [:model/User user {:email "[email protected]"}
:model/Collection {coll-id :id} {:name "Private Collection"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
(let [[model-card dependent-card]
(lib-be/with-metadata-provider-cache
(let [model-card (create-model-card! user "Model Card - permtest" :collection-id coll-id)
@@ -2731,7 +2779,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card)))
;; Admin can access
(is (sequential? (mt/user-http-request :crowberto :get 200
@@ -2745,7 +2793,7 @@
(testing "GET /api/ee/dependencies/graph/broken - does not report transitive breakage"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; Create chain: model-1 -> model-2 -> card
;; When model-1 breaks:
;; - model-2 gets error with source_entity_id=model-1
@@ -2766,7 +2814,7 @@
(break-model-card! model-card-1))
;; Run analysis for both model-2 and dependent-card
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id model-card-2))
(run-analysis-for-card! (:id dependent-card)))
;; Query /graph/broken for model-1 - should only return model-2, not dependent-card
@@ -2781,7 +2829,7 @@
(testing "GET /api/ee/dependencies/graph/broken - filters by dependent-types"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
(let [[model-card dependent-card]
(lib-be/with-metadata-provider-cache
(let [model-card (create-model-card! user "Model Card - deptypestest")
@@ -2790,7 +2838,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-card)))
(testing "filtering by card type returns the dependent card"
(let [response (mt/user-http-request :crowberto :get 200
@@ -2811,7 +2859,7 @@
:model/User creator {:email "[email protected]"}
:model/Collection {personal-coll-id :id} {:personal_owner_id user-id
:name "Test Personal Collection"}]
- (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus :model/AnalysisFinding :model/AnalysisFindingError]
;; one model, two dependents: one in personal collection, one in regular collection
(let [[model-card dependent-in-personal dependent-regular]
(lib-be/with-metadata-provider-cache
@@ -2825,7 +2873,7 @@
(lib-be/with-metadata-provider-cache
(break-model-card! model-card))
(lib-be/with-metadata-provider-cache
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(run-analysis-for-card! (:id dependent-in-personal))
(run-analysis-for-card! (:id dependent-regular)))
(testing "include-personal-collections=false (default) excludes broken dependents in personal collections"
@@ -2845,12 +2893,12 @@
(deftest ^:sequential unreferenced-table-owner-test
(testing "GET /api/ee/dependencies/unreferenced - table owner is returned"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Dependency]
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/Table {table1-id :id} {:name "User Owned Table - ownertest"
:owner_user_id (mt/user->id :crowberto)}
:model/Table {table2-id :id} {:name "Email Owned Table - ownertest"
:owner_email "[email protected]"}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "table" :query "ownertest"
@@ -2871,7 +2919,7 @@
(mt/with-premium-features #{:dependencies}
(let [mp (mt/metadata-provider)
products (lib.metadata/table mp (mt/id :products))]
- (mt/with-model-cleanup [:model/Dependency]
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/Transform {transform1-id :id} {:name "User Owned Transform - ownertest"
:owner_user_id (mt/user->id :crowberto)
:source {:type :query
@@ -2884,7 +2932,7 @@
:query (lib/query mp products)}
:target {:schema "PUBLIC"
:name "email_owned_transform_table"}}]
- (while (#'dependencies.backfill/backfill-dependencies!))
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200
"ee/dependencies/graph/unreferenced"
:types "transform" :query "ownertest"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
index 8435aba1e23..bc182d4d8bb 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
@@ -22,12 +22,12 @@
:measure #{}
:segment #{}
:table #{products-id}}
- (calculation/upstream-deps:card prod-card)))
+ (calculation/calculate-deps :card prod-card)))
(is (= {:card #{prod-card-id}
:measure #{}
:segment #{}
:table #{}}
- (calculation/upstream-deps:card widget-card))))))
+ (calculation/calculate-deps :card widget-card))))))
(deftest ^:parallel upstream-deps-card-join-test
(let [mp (mt/metadata-provider)
@@ -42,12 +42,12 @@
:measure #{}
:segment #{}
:table #{products-id}}
- (calculation/upstream-deps:card products-card)))
+ (calculation/calculate-deps :card products-card)))
(is (= {:card #{products-card-id}
:measure #{}
:segment #{}
:table #{orders-id}}
- (calculation/upstream-deps:card joined-card))))))
+ (calculation/calculate-deps :card joined-card))))))
(deftest ^:parallel upstream-deps-card-implicit-join-fields-test
(let [mp (mt/metadata-provider)
@@ -64,7 +64,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id users-id}}
- (calculation/upstream-deps:card card))))))
+ (calculation/calculate-deps :card card))))))
(deftest ^:parallel upstream-deps-card-implicit-join-filter-test
(let [mp (mt/metadata-provider)
@@ -79,7 +79,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id}}
- (calculation/upstream-deps:card card))))))
+ (calculation/calculate-deps :card card))))))
(deftest ^:parallel upstream-deps-card-implicit-join-breakout-test
(let [mp (mt/metadata-provider)
@@ -94,7 +94,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id}}
- (calculation/upstream-deps:card card))))))
+ (calculation/calculate-deps :card card))))))
(deftest ^:parallel upstream-deps-card-implicit-join-aggregation-test
(let [mp (mt/metadata-provider)
@@ -109,7 +109,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id}}
- (calculation/upstream-deps:card card))))))
+ (calculation/calculate-deps :card card))))))
(deftest ^:parallel upstream-deps-card-implicit-join-order-by-test
(let [mp (mt/metadata-provider)
@@ -124,7 +124,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id}}
- (calculation/upstream-deps:card card))))))
+ (calculation/calculate-deps :card card))))))
(deftest ^:parallel upstream-deps-transform-implicit-join-fields-test
(mt/with-premium-features #{:transforms-basic}
@@ -147,7 +147,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id users-id}}
- (calculation/upstream-deps:transform transform)))))))
+ (calculation/calculate-deps :transform transform)))))))
(deftest ^:parallel upstream-deps-transform-implicit-join-breakout-test
(mt/with-premium-features #{:transforms-basic}
@@ -170,7 +170,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id}}
- (calculation/upstream-deps:transform transform)))))))
+ (calculation/calculate-deps :transform transform)))))))
(deftest ^:parallel upstream-deps-transform-implicit-join-aggregation-test
(mt/with-premium-features #{:transforms-basic}
@@ -192,7 +192,7 @@
:measure #{}
:segment #{}
:table #{checkins-id venues-id}}
- (calculation/upstream-deps:transform transform)))))))
+ (calculation/calculate-deps :transform transform)))))))
(deftest ^:parallel upstream-deps-python-transform-test
(mt/with-premium-features #{:transforms-basic}
@@ -207,7 +207,7 @@
:target {:schema "PUBLIC"
:name "test_output"}}]
(is (= {:table #{products-id orders-id}}
- (calculation/upstream-deps:transform transform)))))))
+ (calculation/calculate-deps :transform transform)))))))
(deftest ^:parallel upstream-deps-native-transform-with-table-tag-test
(testing "GHY-3258: native transform with a table template tag should include the table in dependencies"
@@ -224,7 +224,7 @@
:query query}
:target {:schema "PUBLIC"
:name "test_output"}}]
- (is (=? {:table #(contains? % products-id)} (calculation/upstream-deps:transform transform))))))))
+ (is (=? {:table #(contains? % products-id)} (calculation/calculate-deps :transform transform))))))))
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
(let [mp (mt/metadata-provider)
@@ -255,10 +255,10 @@
:measure #{}
:segment #{}
:table #{products-id}}
- (calculation/upstream-deps:card category-values-card)))
+ (calculation/calculate-deps :card category-values-card)))
(is (= {:card #{category-values-card-id}
:table #{products-id}}
- (calculation/upstream-deps:card native-card))))))
+ (calculation/calculate-deps :card native-card))))))
(deftest ^:parallel upstream-deps-dashboard-test
(let [mp (mt/metadata-provider)
@@ -272,7 +272,7 @@
(t2/hydrate :dashcards))]
(is (= {:card #{card-id}
:dashboard #{}}
- (calculation/upstream-deps:dashboard dashboard)))))))
+ (calculation/calculate-deps :dashboard dashboard)))))))
(deftest ^:parallel upstream-deps-dashboard-with-series-test
(let [mp (mt/metadata-provider)
@@ -288,14 +288,10 @@
:model/DashboardCardSeries _ {:dashboardcard_id dashcard-id
:card_id series-card-id
:position 0}]
- (let [dashcards (t2/select :model/DashboardCard :dashboard_id dashboard-id)
- series-card-ids (when (seq dashcards)
- (t2/select-fn-set :card_id :model/DashboardCardSeries
- :dashboardcard_id [:in (map :id dashcards)]))
- dashboard (assoc dashboard :dashcards dashcards :series-card-ids series-card-ids)]
+ (let [dashboard (t2/hydrate dashboard [:dashcards :series])]
(is (= {:card #{main-card-id series-card-id}
:dashboard #{}}
- (calculation/upstream-deps:dashboard dashboard)))))))
+ (calculation/calculate-deps :dashboard dashboard)))))))
(deftest ^:parallel upstream-deps-dashboard-with-parameter-source-test
(let [mp (mt/metadata-provider)
@@ -317,7 +313,7 @@
(assoc :dashcards dashcards))]
(is (= {:card #{filtered-card-id category-values-card-id}
:dashboard #{}}
- (calculation/upstream-deps:dashboard dashboard)))))))
+ (calculation/calculate-deps :dashboard dashboard)))))))
(deftest ^:parallel upstream-deps-dashboard-with-click-behavior-test
(let [mp (mt/metadata-provider)
@@ -338,7 +334,7 @@
:dashcards dashcards}]
(is (= {:card #{count-card-id target-card-id}
:dashboard #{}}
- (calculation/upstream-deps:dashboard dashboard)))))))
+ (calculation/calculate-deps :dashboard dashboard)))))))
(deftest ^:parallel upstream-deps-dashboard-with-column-click-behavior-test
(let [mp (mt/metadata-provider)
@@ -361,7 +357,7 @@
:dashcards dashcards}]
(is (= {:card #{table-card-id target-card-id}
:dashboard #{}}
- (calculation/upstream-deps:dashboard dashboard)))))))
+ (calculation/calculate-deps :dashboard dashboard)))))))
(deftest ^:parallel upstream-deps-dashboard-with-dashboard-click-behavior-test
(let [mp (mt/metadata-provider)
@@ -382,7 +378,7 @@
:dashcards dashcards}]
(is (= {:card #{count-card-id}
:dashboard #{target-dashboard-id}}
- (calculation/upstream-deps:dashboard dashboard)))))))
+ (calculation/calculate-deps :dashboard dashboard)))))))
(deftest ^:parallel upstream-deps-dashboard-with-column-dashboard-click-behavior-test
(let [mp (mt/metadata-provider)
@@ -405,7 +401,7 @@
:dashcards dashcards}]
(is (= {:card #{table-card-id}
:dashboard #{target-dashboard-id}}
- (calculation/upstream-deps:dashboard dashboard)))))))
+ (calculation/calculate-deps :dashboard dashboard)))))))
(deftest ^:parallel upstream-deps-document-test
(let [mp (mt/metadata-provider)
@@ -431,7 +427,7 @@
(is (= {:card #{card-id embedded-card-id}
:dashboard #{dashboard-id}
:table #{products-id}}
- (calculation/upstream-deps:document document))))))
+ (calculation/calculate-deps :document document))))))
(deftest upstream-deps-sandbox-test
(mt/with-premium-features #{:sandboxes}
@@ -441,7 +437,7 @@
:table_id (mt/id :products)
:card_id sandbox-card-id}]
(is (= {:card #{sandbox-card-id}}
- (calculation/upstream-deps:sandbox sandbox))))))
+ (calculation/calculate-deps :sandbox sandbox))))))
(deftest ^:parallel upstream-deps-segment-test
(let [products-id (mt/id :products)
@@ -451,7 +447,7 @@
(mt/with-temp [:model/Segment segment {:table_id products-id
:definition {:filter [:> [:field price-field-id nil] 50]}}]
(is (= {:segment #{} :table #{products-id}}
- (calculation/upstream-deps:segment segment)))))
+ (calculation/calculate-deps :segment segment)))))
(testing "segment depending on another segment"
(mt/with-temp [:model/Segment {segment-a-id :id :as segment-a} {:table_id products-id
@@ -462,11 +458,11 @@
[:= [:field category-field-id nil] "Widget"]]}}]
(testing "base segment depends only on table"
(is (= {:segment #{} :table #{products-id}}
- (calculation/upstream-deps:segment segment-a))))
+ (calculation/calculate-deps :segment segment-a))))
(testing "dependent segment depends on both table and segment"
(is (= {:table #{products-id}
:segment #{segment-a-id}}
- (calculation/upstream-deps:segment segment-b))))))))
+ (calculation/calculate-deps :segment segment-b))))))))
(deftest ^:parallel upstream-deps-card-with-multiple-segments-test
(testing "Card using multiple segments depends on all of them"
@@ -487,7 +483,7 @@
:measure #{}
:segment #{segment-a-id segment-b-id}
:table #{products-id}}
- (calculation/upstream-deps:card card)))))))
+ (calculation/calculate-deps :card card)))))))
(deftest ^:parallel upstream-deps-segment-with-multiple-segments-test
(testing "Segment depending on multiple other segments tracks all dependencies"
@@ -506,7 +502,7 @@
[:> [:field rating-field-id nil] 4]]}}]
(is (= {:table #{products-id}
:segment #{segment-a-id segment-b-id}}
- (calculation/upstream-deps:segment segment-c)))))))
+ (calculation/calculate-deps :segment segment-c)))))))
(deftest ^:parallel upstream-deps-segment-implicit-join-test
(testing "Segment depending on implicitly joined field adds dep on that field's table"
@@ -518,7 +514,7 @@
:definition {:filter [:= [:field venue-name-field-id {:source-field venue-fk-field-id}] "Bird's Nest"]}}]
(is (= {:segment #{}
:table #{checkins-id venues-id}}
- (calculation/upstream-deps:segment segment)))))))
+ (calculation/calculate-deps :segment segment)))))))
(deftest upstream-deps-measure-test
(let [mp (mt/metadata-provider)
@@ -531,7 +527,7 @@
:definition (-> (lib/query mp orders)
(lib/aggregate (lib/sum quantity)))}]
(is (= {:measure #{} :segment #{} :table #{orders-id}}
- (calculation/upstream-deps:measure measure)))))
+ (calculation/calculate-deps :measure measure)))))
(testing "measure depending on another measure"
(mt/with-temp [:model/Measure {measure-a-id :id :as measure-a} {:name "Measure A"
@@ -545,10 +541,10 @@
(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id) 100)))}]
(testing "base measure depends only on table"
(is (= {:measure #{} :segment #{} :table #{orders-id}}
- (calculation/upstream-deps:measure measure-a))))
+ (calculation/calculate-deps :measure measure-a))))
(testing "dependent measure depends on both table and measure"
(is (= {:measure #{measure-a-id} :segment #{} :table #{orders-id}}
- (calculation/upstream-deps:measure measure-b))))))))))
+ (calculation/calculate-deps :measure measure-b))))))))))
(deftest upstream-deps-measure-with-multiple-measures-test
(testing "Measure depending on multiple other measures tracks all dependencies"
@@ -572,7 +568,7 @@
(lib/aggregate (lib/+ (lib.metadata/measure mp' measure-a-id)
(lib.metadata/measure mp' measure-b-id))))}]
(is (= {:measure #{measure-a-id measure-b-id} :segment #{} :table #{orders-id}}
- (calculation/upstream-deps:measure measure-c)))))))))
+ (calculation/calculate-deps :measure measure-c)))))))))
(deftest upstream-deps-measure-implicit-join-test
(testing "Measure depending on implicitly joined field adds dep on that field's table"
@@ -587,7 +583,7 @@
:table_id checkins-id
:definition (lib/aggregate base-query (lib/sum venue-price))}]
(is (= {:measure #{} :segment #{} :table #{checkins-id venues-id}}
- (calculation/upstream-deps:measure measure)))))))
+ (calculation/calculate-deps :measure measure)))))))
(deftest upstream-deps-card-with-measure-test
(testing "Card using a measure depends on that measure"
@@ -603,7 +599,7 @@
(mt/with-temp [:model/Card card {:dataset_query (-> (lib/query mp' orders)
(lib/aggregate (lib.metadata/measure mp' measure-id)))}]
(is (= {:card #{} :measure #{measure-id} :segment #{} :table #{orders-id}}
- (calculation/upstream-deps:card card)))))))))
+ (calculation/calculate-deps :card card)))))))))
(deftest upstream-deps-measure-with-segment-test
(testing "Measure with conditional aggregation using segment depends on that segment"
@@ -621,4 +617,4 @@
:definition (-> (lib/query mp' orders)
(lib/aggregate (lib/sum-where quantity (lib/ref segment-meta))))}]
(is (= {:measure #{} :segment #{segment-id} :table #{orders-id}}
- (calculation/upstream-deps:measure measure)))))))))
+ (calculation/calculate-deps :measure measure)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index 897d692b86b..27fc7016d87 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -1,20 +1,18 @@
(ns metabase-enterprise.dependencies.events-test
(:require
- [clojure.string :as str]
[clojure.test :refer [deftest is testing]]
- [metabase-enterprise.dependencies.async :as async]
- [metabase-enterprise.dependencies.calculation :as deps.calculation]
[metabase-enterprise.dependencies.events]
[metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
- [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.api.common :as api]
[metabase.events.core :as events]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.test :as mt]
- [metabase.util.log.capture :as log.capture]
[toucan2.core :as t2]))
(comment
@@ -23,11 +21,17 @@
(defn- run-with-dependencies-setup! [thunk]
(mt/with-test-user :rasta
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/AnalysisFinding]
+ (mt/with-model-cleanup [:model/AnalysisFinding :model/DependencyStatus]
(lib-be/with-metadata-provider-cache
(let [mp (mt/metadata-provider)]
(thunk mp)))))))
+(defn- assert-stale
+ "Assert that the given entity is marked stale in dependency_status."
+ [entity-type entity-id]
+ (is (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true)
+ (str "Expected " (name entity-type) " " entity-id " to be marked stale")))
+
(deftest dashboard-update-sets-correct-dependencies
(run-with-dependencies-setup!
(fn [mp]
@@ -51,6 +55,8 @@
:card_id series-card-id
:position 0}]
(events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})
+ (assert-stale :dashboard dashboard-id)
+ (deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :dashboard
:from_entity_id dashboard-id
:to_entity_type :card
@@ -107,6 +113,7 @@
:targetId column-click-target-dashboard-id}}}}}]
(let [updated-dashboard (t2/select-one :model/Dashboard :id dashboard-id)]
(events/publish-event! :event/dashboard-update {:object updated-dashboard :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :dashboard
:from_entity_id dashboard-id
:to_entity_type :card
@@ -163,6 +170,8 @@
{:type "cardEmbed"
:attrs {:id embedded-card-id}}]}}]
(events/publish-event! :event/document-create {:object document :user-id api/*current-user-id*})
+ (assert-stale :document document-id)
+ (deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :document
:from_entity_id document-id
:to_entity_type :card
@@ -183,6 +192,7 @@
:model "table"}}]}]})]
(t2/update! :model/Document document-id updated-doc)
(events/publish-event! :event/document-update {:object updated-doc :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :document
:from_entity_id document-id
:to_entity_type :dashboard
@@ -208,6 +218,8 @@
:table_id (mt/id :products)
:card_id card1-id}]
(events/publish-event! :event/sandbox-create {:object sandbox :user-id api/*current-user-id*})
+ (assert-stale :sandbox sandbox-id)
+ (deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :sandbox
:from_entity_id sandbox-id
:to_entity_type :card
@@ -217,6 +229,7 @@
(t2/update! :model/Sandbox sandbox-id (assoc sandbox :card_id card2-id))
(let [updated-sandbox (t2/select-one :model/Sandbox :id sandbox-id)]
(events/publish-event! :event/sandbox-update {:object updated-sandbox :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :sandbox
:from_entity_id sandbox-id
:to_entity_type :card
@@ -227,64 +240,44 @@
(events/publish-event! :event/sandbox-delete {:object sandbox :user-id api/*current-user-id*})
(is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))))))
-(deftest card-dependency-calculation-error-handling-test
- (testing "When dependency calculation throws an error, it should be logged and the version should still be updated"
+(deftest card-marks-stale-on-create-update-test
+ (testing "Card create/update events mark the entity stale in dependency_status"
(run-with-dependencies-setup!
(fn [mp]
- (let [products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
+ (let [products (lib.metadata/table mp (mt/id :products))]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:card (fn [_]
- (throw (ex-info "Dependency calculation failed"
- {:card-id card-id})))]
- (testing "on create event"
- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})
- (is (some #(and (= "Dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %))
- (str/includes? (:message %) "{:entity-type :card")
- (str/includes? (:message %) (str card-id)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Card :id card-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))
- (testing "on update event"
- (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})
- (is (some #(and (= "Dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %))
- (str/includes? (:message %) "{:entity-type :card")
- (str/includes? (:message %) (str card-id)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Card :id card-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))))))))))
-
-(deftest snippet-dependency-calculation-error-handling-test
- (testing "When snippet dependency calculation throws an error, it should be logged and the version should still be updated"
+ (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})
+ (assert-stale :card card-id)
+ (deps.test/synchronously-run-backfill!)
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :products)))
+ (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})
+ (assert-stale :card card-id)))))))
+
+(deftest snippet-marks-stale-on-create-update-test
+ (testing "Snippet create/update events mark the entity stale in dependency_status"
(run-with-dependencies-setup!
(fn [_]
(mt/with-temp [:model/NativeQuerySnippet {snippet-id :id :as snippet} {:name "test snippet"
:content "SELECT 1"}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:snippet (fn [_]
- (throw (ex-info "Snippet dependency calculation failed"
- {:snippet-id snippet-id})))]
- (testing "on create event"
- (events/publish-event! :event/snippet-create {:object snippet :user-id api/*current-user-id*})
- (is (some #(and (= "Snippet dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet :id snippet-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id snippet-id :from_entity_type :snippet))))
- (testing "on update event"
- (events/publish-event! :event/snippet-update {:object snippet :user-id api/*current-user-id*})
- (is (some #(and (= "Snippet dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet :id snippet-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id snippet-id :from_entity_type :snippet)))))))))))
+ (events/publish-event! :event/snippet-create {:object snippet :user-id api/*current-user-id*})
+ (assert-stale :snippet snippet-id)
+ (events/publish-event! :event/snippet-update {:object snippet :user-id api/*current-user-id*})
+ (assert-stale :snippet snippet-id))))))
+
+(deftest mark-stale-failure-does-not-propagate-test
+ (testing "When mark-stale! throws, the event handler catches the error and the API call is not affected"
+ (run-with-dependencies-setup!
+ (fn [mp]
+ (let [products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]
+ (with-redefs [deps.dependency-status/mark-stale!
+ (fn [_ _] (throw (ex-info "Simulated DB failure" {})))]
+ ;; Should not throw — the error is caught and logged
+ (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*}))
+ ;; Entity should NOT be stale (mark-stale! failed)
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
+ "Entity should not be marked stale when mark-stale! fails")))))))
(deftest ^:sequential native-transform-updates-dependencies-test
(testing "native transform update events trigger dependency calculations"
@@ -294,8 +287,8 @@
:type :query}]
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
(events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (assert-stale :transform transform-id)
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
:to_entity_type :table,
@@ -310,8 +303,8 @@
:type :query}]
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
(events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (assert-stale :transform transform-id)
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
:to_entity_type :table,
@@ -333,8 +326,8 @@
"import pandas as pd\n\ndef transform(orders):\n return orders"}]
(mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
(events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
+ (assert-stale :transform transform-id)
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
:to_entity_type :table,
@@ -355,8 +348,6 @@
:output-table :test_table
:transform-id transform-id}
:user-id api/*current-user-id*})
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
@@ -398,17 +389,21 @@
(events/publish-event! :event/update-transform
{:object transform
:user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
:to_entity_id transform-id}]
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "changing target update"
- (events/publish-event! :event/update-transform
- {:object (assoc-in transform [:target :name] "test_table2")
- :user-id api/*current-user-id*})
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
+ (let [updated (t2/select-one :model/Transform :id transform-id)]
+ (events/publish-event! :event/update-transform
+ {:object updated
+ :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))
(deftest ^:sequential query-transform-update-handles-downstream-dependencies-test
(testing "query transform update events handles downstream dependencies"
@@ -437,127 +432,21 @@
(events/publish-event! :event/update-transform
{:object transform
:user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
:to_entity_id transform-id}]
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "changing target update"
- (events/publish-event! :event/update-transform
- {:object (assoc-in transform [:target :name] "test_table2")
- :user-id api/*current-user-id*})
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
-
-(deftest transform-dependency-calculation-error-handling-test
- (testing "When transform dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup!
- (fn [_]
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:transform (fn [_]
- (throw (ex-info "Transform dependency calculation failed"
- {:transform-id transform-id})))]
- (testing "on create event"
- (events/publish-event! :event/create-transform {:object transform :user-id api/*current-user-id*})
- (is (some #(and (= "Transform dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform))))
- (testing "on update event"
- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (is (some #(and (= "Transform dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform :id transform-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))))
-
-(deftest dashboard-dependency-calculation-error-handling-test
- (testing "When dashboard dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup!
- (fn [_]
- (mt/with-temp [:model/Dashboard {dashboard-id :id :as dashboard} {}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:dashboard (fn [_]
- (throw (ex-info "Dashboard dependency calculation failed"
- {:dashboard-id dashboard-id})))]
- (testing "on create event"
- (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})
- (is (some #(and (= "Dashboard dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Dashboard :id dashboard-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard))))
- (testing "on update event"
- (events/publish-event! :event/dashboard-update {:object dashboard :user-id api/*current-user-id*})
- (is (some #(and (= "Dashboard dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Dashboard :id dashboard-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))))))))
-
-(deftest document-dependency-calculation-error-handling-test
- (testing "When document dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup!
- (fn [_]
- (mt/with-temp [:model/Document {document-id :id :as document} {:content_type "application/json+vnd.prose-mirror"
- :document {:type "doc" :content []}}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:document (fn [_]
- (throw (ex-info "Document dependency calculation failed"
- {:document-id document-id})))]
- (testing "on create event"
- (events/publish-event! :event/document-create {:object document :user-id api/*current-user-id*})
- (is (some #(and (= "Document dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Document :id document-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document))))
- (testing "on update event"
- (events/publish-event! :event/document-update {:object document :user-id api/*current-user-id*})
- (is (some #(and (= "Document dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Document :id document-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document)))))))))))
-
-(deftest sandbox-dependency-calculation-error-handling-test
- (testing "When sandbox dependency calculation throws an error, it should be logged and the version should still be updated"
- (mt/with-premium-features #{:sandboxes}
- (run-with-dependencies-setup!
- (fn [_]
- (mt/with-temp [:model/PermissionsGroup {group-id :id} {:name "test group"}
- :model/Card {card-id :id} {}
- :model/Sandbox {sandbox-id :id :as sandbox} {:group_id group-id
- :table_id (mt/id :products)
- :card_id card-id}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:sandbox (fn [_]
- (throw (ex-info "Sandbox dependency calculation failed"
- {:sandbox-id sandbox-id})))]
- (testing "on create event"
- (events/publish-event! :event/sandbox-create {:object sandbox :user-id api/*current-user-id*})
- (is (some #(and (= "Sandbox dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Sandbox :id sandbox-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))
- (testing "on update event"
- (events/publish-event! :event/sandbox-update {:object sandbox :user-id api/*current-user-id*})
- (is (some #(and (= "Sandbox dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Sandbox :id sandbox-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))))))
+ (t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
+ (let [updated (t2/select-one :model/Transform :id transform-id)]
+ (events/publish-event! :event/update-transform
+ {:object updated
+ :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))))
(deftest segment-update-sets-correct-dependencies
(run-with-dependencies-setup!
@@ -569,6 +458,8 @@
:definition {:filter [:> [:field price-field-id nil] 50]}}]
(testing "creating a segment creates dependency to its table"
(events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})
+ (assert-stale :segment segment-id)
+ (deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :segment
:from_entity_id segment-id
:to_entity_type :table
@@ -579,6 +470,7 @@
(t2/update! :model/Segment segment-id {:definition {:filter [:= [:field category-field-id nil] "Widget"]}})
(let [updated-segment (t2/select-one :model/Segment :id segment-id)]
(events/publish-event! :event/segment-update {:object updated-segment :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :segment
:from_entity_id segment-id
:to_entity_type :table
@@ -599,11 +491,13 @@
:definition {:filter [:> [:field price-field-id nil] 50]}}]
(testing "creating a card using a segment creates dependencies to both segment and table"
(events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query {:database (mt/id)
:type :query
:query {:source-table products-id
:filter [:segment segment-id]}}}]
(events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :card
:from_entity_id card-id
:to_entity_type :segment
@@ -615,35 +509,6 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card))))))))))))
-(deftest segment-dependency-calculation-error-handling-test
- (testing "When segment dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup!
- (fn [_]
- (let [products-id (mt/id :products)
- price-field-id (mt/id :products :price)]
- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id
- :definition {:filter [:> [:field price-field-id nil] 50]}}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:segment (fn [_]
- (throw (ex-info "Segment dependency calculation failed"
- {:segment-id segment-id})))]
- (testing "on create event"
- (events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})
- (is (some #(and (= "Segment dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Segment :id segment-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))
- (testing "on update event"
- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})
- (is (some #(and (= "Segment dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Segment :id segment-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))))))))
-
(deftest measure-update-sets-correct-dependencies
(run-with-dependencies-setup!
(fn [mp]
@@ -657,6 +522,8 @@
(lib/aggregate (lib/sum quantity)))}]
(testing "creating a measure creates dependency to its table"
(events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})
+ (assert-stale :measure measure-id)
+ (deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure-id
:to_entity_type :table
@@ -668,6 +535,7 @@
(lib/aggregate (lib/sum subtotal)))})
(let [updated-measure (t2/select-one :model/Measure :id measure-id)]
(events/publish-event! :event/measure-update {:object updated-measure :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure-id
:to_entity_type :table
@@ -697,6 +565,7 @@
(testing "creating a measure that references another measure creates dependencies to both"
(events/publish-event! :event/measure-create {:object measure1 :user-id api/*current-user-id*})
(events/publish-event! :event/measure-create {:object measure2 :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure2-id
:to_entity_type :table
@@ -718,12 +587,14 @@
:definition (-> (lib/query mp orders)
(lib/aggregate (lib/sum quantity)))}]
(events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(testing "creating a card using a measure creates dependencies to both measure and table"
(let [mp' (mt/metadata-provider)
query (-> (lib/query mp' orders)
(lib/aggregate (lib.metadata/measure mp' measure-id)))]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query query}]
(events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :card
:from_entity_id card-id
:to_entity_type :measure
@@ -735,36 +606,7 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id card-id :from_entity_type :card)))))))))))))
-(deftest measure-dependency-calculation-error-handling-test
- (testing "When measure dependency calculation throws an error, it should be logged and the version should still be updated"
- (run-with-dependencies-setup!
- (fn [mp]
- (let [orders-id (mt/id :orders)
- orders (lib.metadata/table mp orders-id)
- quantity (lib.metadata/field mp (mt/id :orders :quantity))]
- (mt/with-temp [:model/Measure {measure-id :id :as measure} {:table_id orders-id
- :definition (-> (lib/query mp orders)
- (lib/aggregate (lib/sum quantity)))}]
- (log.capture/with-log-messages-for-level [messages ["metabase-enterprise.dependencies.events" :error]]
- (with-redefs [deps.calculation/upstream-deps:measure (fn [_]
- (throw (ex-info "Measure dependency calculation failed"
- {:measure-id measure-id})))]
- (testing "on create event"
- (events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})
- (is (some #(and (= "Measure dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Measure :id measure-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))
- (testing "on update event"
- (events/publish-event! :event/measure-update {:object measure :user-id api/*current-user-id*})
- (is (some #(and (= "Measure dependency calculation failed" (ex-message (:e %)))
- (= :error (:level %)))
- (messages)))
- (is (= models.dependency/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Measure :id measure-id)))
- (is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))))
+;; === Analysis finding tests (these are unchanged — they use AnalysisFinding, not dependency_analysis_version) ===
(defn- assert-has-analyses
"Asserts that a given set of entities have the appropriate analysis version.
@@ -788,7 +630,7 @@
(deftest ^:sequential card-update-works-with-no-analyses-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Card updates should analyze the card and mark dependents stale for async processing"
+ (testing "Card updates should mark the card stale and trigger the background job"
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
(mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
@@ -797,10 +639,10 @@
:from_entity_id child-card-id
:to_entity_type :card
:to_entity_id parent-card-id}]
- (with-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))
+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ ;; No analysis record should exist yet - mark-entity-stale! is a no-op for never-analyzed entities
(assert-has-analyses
- {:card {parent-card-id -1}})
+ {:card {parent-card-id nil}})
(is (nil? (t2/select-one-fn :analysis_version :model/AnalysisFinding
:analyzed_entity_type :card
:analyzed_entity_id child-card-id))
@@ -809,13 +651,12 @@
(deftest ^:sequential card-update-updates-analyses-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Card updates should update the card's analysis and mark dependents stale"
+ (testing "Card updates should mark the card stale, then the job re-analyzes and propagates"
(let [products-id (mt/id :products)
orders-id (mt/id :orders)
products (lib.metadata/table mp products-id)
orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]
+ old-version models.analysis-finding/*current-analysis-finding-version*]
(mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
:model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
@@ -831,83 +672,41 @@
{:card {parent-card-id old-version
child-card-id old-version
other-card-id old-version}}))
- (t2/with-transaction [_conn]
- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (with-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})))
- (testing "Parent should be re-analyzed synchronously"
+ (testing "card update only marks direct dependents stale - not its own analysis"
+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ (assert-has-analyses
+ {:card {parent-card-id old-version
+ child-card-id -1
+ other-card-id old-version}}))
+ (testing "If the version changes, the job picks up only the stale cards"
+ (binding [models.analysis-finding/*current-analysis-finding-version* (inc old-version)]
(assert-has-analyses
- {:card {parent-card-id new-version}}))
- (testing "Child should be marked stale (not re-analyzed yet)"
- (let [child-finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id child-card-id)]
- (is (= old-version (:analysis_version child-finding)))
- (is (true? (:stale child-finding)))))
- (testing "Other card should be unchanged"
- (let [other-finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id other-card-id)]
- (is (= old-version (:analysis_version other-finding)))
- (is (false? (:stale other-finding))))))))))))
+ {:card {parent-card-id old-version
+ child-card-id -1
+ other-card-id old-version}})))))))))
-(deftest ^:sequential stale-entities-are-processed-by-job-test
- (run-with-dependencies-setup!
- (fn [mp]
- (testing "Stale entities are re-analyzed when the job runs"
- (let [products (lib.metadata/table mp (mt/id :products))
- current-version models.analysis-finding/*current-analysis-finding-version*]
- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
- :model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-card-id
- :to_entity_type :card
- :to_entity_id parent-card-id}]
- ;; Create initial analyses for both cards
- (deps.findings/upsert-analysis! parent-card)
- (deps.findings/upsert-analysis! child-card)
- ;; Update parent - this marks child as stale
- (t2/with-transaction [_conn]
- (with-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*}))
- ;; Verify child is stale
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id child-card-id))
- "Child should be marked stale after parent update"))
- ;; Run the job to process stale entities
- (deps.findings/analyze-batch! :card 10)
- ;; Verify child is no longer stale and has been re-analyzed
- (let [child-finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id child-card-id)]
- (is (false? (:stale child-finding))
- "Child should no longer be stale after job runs")
- (is (= current-version (:analysis_version child-finding))
- "Child should have current analysis version"))))))))
+;;; ------------------------------------------------ Analysis propagation tests ------------------------------------------------
(deftest ^:sequential card-update-transaction-rollback-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "If marking dependents stale fails, analysis upsert is rolled back"
+ (testing "If marking immediate dependents stale fails, analysis upsert is rolled back"
(let [products (lib.metadata/table mp (mt/id :products))
old-version models.analysis-finding/*current-analysis-finding-version*
new-version (inc old-version)]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]
- ;; Create initial analysis
(deps.findings/upsert-analysis! card)
(testing "Initial analysis exists"
(is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding
:analyzed_entity_type :card
:analyzed_entity_id card-id))))
- ;; Try to update with mark-dependents-stale! throwing an exception
(binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (with-redefs [async/submit! (fn [f] (f))
- deps.findings/mark-dependents-stale! (fn [_ _] (throw (ex-info "Simulated failure" {})))]
+ (with-redefs [deps.findings/mark-immediate-dependents-stale!
+ (fn [_ _] (throw (ex-info "Simulated failure" {})))]
+ ;; analyze-and-propagate! wraps in a transaction, so the upsert should be rolled back
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated failure"
- (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})))))
- ;; Verify analysis was NOT updated (rolled back)
- (testing "Analysis should be unchanged after failed event"
+ (#'deps.findings/analyze-and-propagate! card)))))
+ (testing "Analysis should be unchanged after rolled-back transaction"
(is (= old-version (t2/select-one-fn :analysis_version :model/AnalysisFinding
:analyzed_entity_type :card
:analyzed_entity_id card-id))))))))))
@@ -915,48 +714,37 @@
(deftest ^:sequential card-update-triggers-native-cards-test
(run-with-dependencies-setup!
(fn [mp]
- (let [old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc old-version)]
- (mt/with-temp [:model/Card
- {parent-id :id :as parent}
- {:dataset_query (lib/native-query mp "select * from products")}
-
- :model/Card
- {child-id :id :as child}
- {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
-
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}]
+ (testing "Card update marks entity stale, and the entity-check job re-analyzes it and marks dependents stale"
+ (mt/with-temp [:model/Card {parent-id :id :as parent} {:dataset_query (lib/native-query mp "select * from products")}
+ :model/Card {child-id :id :as child} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id child-id
+ :to_entity_type :card :to_entity_id parent-id}]
(deps.findings/upsert-analysis! parent)
(deps.findings/upsert-analysis! child)
- (t2/with-transaction [_conn]
- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (with-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update {:object parent
- :previous-object parent
- :user-id api/*current-user-id*})
- (testing "Native card update should trigger analysis"
- (assert-has-analyses
- {:card {parent-id new-version}}))
- (testing "Its dependents should be marked stale"
- (let [child-finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id child-id)]
- (is (= old-version (:analysis_version child-finding)))
- (is (true? (:stale child-finding)))))))))))))
+ ;; Event marks entity stale in analysis_finding
+ (events/publish-event! :event/card-update {:object parent :previous-object parent :user-id api/*current-user-id*})
+ (testing "Parent card should be marked stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id parent-id))))
+ ;; Run entity-check job to process stale entities
+ (#'task.entity-check/check-entities!)
+ (testing "Parent should be re-analyzed"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id parent-id))))
+ (testing "Child should have been re-analyzed via wave propagation"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id child-id)))))))))
(deftest ^:sequential card-update-stops-on-transforms-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Card updates should continue analyzing through a transform"
- (let [products-id (mt/id :products)
- orders-id (mt/id :orders)
- products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]
+ (testing "Card update marks immediate dependents stale including transforms, wave propagates through"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))
+ old-version models.analysis-finding/*current-analysis-finding-version*]
(mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
:model/Transform {transform-id :id :as transform} {:source {:type :query
:query (lib/query mp products)}
@@ -965,137 +753,91 @@
:name "sample"
:type :table}}
:model/Card {child-card-id :id :as child-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-card-id
- :to_entity_type :transform
- :to_entity_id transform-id}
- :model/Dependency _ {:from_entity_type :transform
- :from_entity_id transform-id
- :to_entity_type :card
- :to_entity_id parent-card-id}]
+ :model/Dependency _ {:from_entity_type :transform :from_entity_id transform-id
+ :to_entity_type :card :to_entity_id parent-card-id}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id child-card-id
+ :to_entity_type :transform :to_entity_id transform-id}]
(deps.findings/upsert-analysis! parent-card)
(deps.findings/upsert-analysis! transform)
(deps.findings/upsert-analysis! child-card)
- (testing "Checking that initial analyses exist"
- (assert-has-analyses
- {:card {parent-card-id old-version
- child-card-id old-version}
- :transform {transform-id old-version}}))
- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (t2/with-transaction [_conn]
- (with-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
- (testing "parent card should be re-analyzed synchronously"
- (is (=? {:analysis_version new-version :stale false}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id parent-card-id))))
- (testing "transform (direct dependent) should be marked stale"
- (is (=? {:analysis_version old-version :stale true}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id))))
- (testing "child card (transitive dependent) should be marked stale"
- (is (=? {:analysis_version old-version :stale true}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id child-card-id)))))))))))))
+ (assert-has-analyses
+ {:card {parent-card-id old-version child-card-id old-version}
+ :transform {transform-id old-version}})
+ ;; Event marks parent stale in analysis_finding, which triggers entity-check
+ (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ ;; Run entity-check job — should propagate through transform to child via waves
+ (#'task.entity-check/check-entities!)
+ (testing "All entities should be re-analyzed after wave propagation"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id parent-card-id)))
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :transform :analyzed_entity_id transform-id)))
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id child-card-id))))))))))
(deftest ^:sequential transform-update-works-with-no-analyses-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "transform update creates new analyses"
- (let [products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id :as transform} {:source {:type :query
- :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :transform
- :to_entity_id transform-id}]
- (events/publish-event! :event/transform-update {:object transform :user-id api/*current-user-id*})
- (assert-has-analyses
- ;; child cards shouldn't be updated on transform update, only on run
- {:card {card-id nil}
- :transform {transform-id -1}})))))))
+ (testing "Transform update does not error when no analyses exist yet"
+ (let [products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-temp [:model/Transform transform {:source {:type :query
+ :query (lib/query mp products)}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ ;; Should not throw even when no analysis findings exist
+ (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})))))))))
(deftest ^:sequential transform-update-updates-analyses-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "transform update updates analyses when they exist"
- (let [products-id (mt/id :products)
- orders-id (mt/id :orders)
- products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id :as transform} {:source {:type :query
- :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}
- :model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :transform
- :to_entity_id transform-id}]
- (deps.findings/upsert-analysis! card)
- (deps.findings/upsert-analysis! transform)
- (deps.findings/upsert-analysis! other-card)
- (testing "Checking that initial analyses exist"
- (assert-has-analyses
- {:card {card-id old-version
- other-card-id old-version}
- :transform {transform-id old-version}}))
- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*}))
- (testing "Checking that the correct analyses were updated"
- (assert-has-analyses
- ;; neither unrelated cards nor dependent cards should be updated on transform update
- {:card {card-id old-version
- other-card-id old-version}
- :transform {transform-id new-version}})))))))))
+ (testing "Transform update marks entity stale, entity-check job re-analyzes it"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ old-version models.analysis-finding/*current-analysis-finding-version*]
+ (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query
+ :query (lib/query mp products)}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ (deps.findings/upsert-analysis! transform)
+ (assert-has-analyses {:transform {transform-id old-version}})
+ (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
+ (testing "Transform should be marked stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :transform
+ :analyzed_entity_id transform-id))))
+ (#'task.entity-check/check-entities!)
+ (testing "Transform should be re-analyzed"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :transform
+ :analyzed_entity_id transform-id))))))))))
(deftest ^:sequential transform-update-triggers-native-transforms-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Native transform updates should trigger analysis"
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source {:type :query
- :query (lib/native-query mp "select * from products")}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}]
- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (assert-has-analyses
- {:transform {transform-id models.analysis-finding/*current-analysis-finding-version*}}))))))
+ (testing "Native transform updates do not error"
+ (mt/with-temp [:model/Transform transform {:source {:type :query
+ :query (lib/native-query mp "select * from products")}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ ;; Should not throw even for native transforms
+ (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*}))))))))
(deftest ^:sequential transform-run-works-with-no-analyses-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "transform runs work when dependent cards have no pre-existing analyses"
- (let [products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
+ (testing "Transform run completes without error when no analyses exist"
+ (let [products (lib.metadata/table mp (mt/id :products))]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id} {:source {:type :query
- :query (lib/query mp products)}
+ :model/Transform {transform-id :id} {:source {:type :query :query (lib/query mp products)}
:name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :transform
- :to_entity_id transform-id}]
- ;; Event should complete without error even when there are no pre-existing analyses
+ :target {:schema "public" :name "sample" :type :table}}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :transform :to_entity_id transform-id}]
(is (some? (events/publish-event! :event/transform-run-complete
{:object {:db-id (mt/id)
:output-schema "public"
@@ -1103,152 +845,99 @@
:transform-id transform-id}
:user-id api/*current-user-id*})))))))))
-(deftest ^:sequential transform-run-updates-analyses-test
+(deftest ^:sequential transform-run-marks-dependents-stale-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "transform runs update existing analyses when they exist"
- (let [products-id (mt/id :products)
- orders-id (mt/id :orders)
- products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*]
+ (testing "Transform run marks immediate dependents stale"
+ (let [products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
- :model/Transform {transform-id :id :as transform} {:source {:type :query
- :query (lib/query mp products)}
+ :model/Transform {transform-id :id :as transform} {:source {:type :query :query (lib/query mp products)}
:name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}
+ :target {:schema "public" :name "sample" :type :table}}
:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :transform
- :to_entity_id transform-id}]
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :transform :to_entity_id transform-id}]
(deps.findings/upsert-analysis! card)
(deps.findings/upsert-analysis! transform)
(deps.findings/upsert-analysis! other-card)
- (testing "checking that initial analyses exist"
- (assert-has-analyses
- {:card {card-id old-version
- other-card-id old-version}
- :transform {transform-id old-version}}))
- (t2/with-transaction [_conn]
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "public"
- :output-table "sample"
- :transform-id transform-id}
- :user-id api/*current-user-id*})
- (testing "dependent card should be marked stale"
- (is (=? {:analysis_version old-version :stale true}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id))))
- (testing "other card should be unchanged"
- (is (=? {:analysis_version old-version :stale false}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id other-card-id))))
- (testing "transform should be unchanged"
- (is (=? {:analysis_version old-version :stale false}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :transform
- :analyzed_entity_id transform-id)))))))))))
+ (events/publish-event! :event/transform-run-complete
+ {:object {:db-id (mt/id)
+ :output-schema "public"
+ :output-table "sample"
+ :transform-id transform-id}
+ :user-id api/*current-user-id*})
+ (testing "Dependent card should be marked stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id card-id))))
+ (testing "Other card should be unchanged"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id other-card-id))))
+ (testing "Transform should be unchanged"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :transform :analyzed_entity_id transform-id))))))))))
(deftest ^:sequential segment-update-works-with-no-analyses-test
(run-with-dependencies-setup!
- (fn [mp]
- (testing "segment update creates analysis for the segment itself"
+ (fn [_]
+ (testing "Segment update does not error when no analyses exist yet"
(let [products-id (mt/id :products)
- price-field-id (mt/id :products :price)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id
- :definition {:filter [:> [:field price-field-id nil] 50]}}
- :model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :segment
- :to_entity_id segment-id}]
- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})
- ;; Segment should be analyzed synchronously
- (assert-has-analyses
- {:segment {segment-id -1}})))))))
+ price-field-id (mt/id :products :price)]
+ (mt/with-temp [:model/Segment segment {:table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 50]}}]
+ ;; Should not throw even when no analysis findings exist
+ (is (some? (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})))))))))
(deftest ^:sequential segment-update-updates-analyses-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "segment update updates analyses when they exist"
+ (testing "Segment update marks entity stale, entity-check job re-analyzes and propagates"
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)
products (lib.metadata/table mp products-id)
- old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]
+ old-version models.analysis-finding/*current-analysis-finding-version*]
(mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id
:definition {:filter [:> [:field price-field-id nil] 50]}}
:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :segment
- :to_entity_id segment-id}]
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :segment :to_entity_id segment-id}]
(deps.findings/upsert-analysis! segment)
(deps.findings/upsert-analysis! card)
- (testing "checking that initial analyses exist"
- (assert-has-analyses
- {:card {card-id old-version}
- :segment {segment-id old-version}}))
- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (t2/with-transaction [_conn]
- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})
- (testing "segment should be re-analyzed synchronously"
- (is (=? {:analysis_version new-version :stale false}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :segment
- :analyzed_entity_id segment-id))))
- (testing "dependent card should be marked stale"
- (is (=? {:analysis_version old-version :stale true}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id))))))))))))
+ (assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})
+ (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})
+ (testing "Segment should be marked stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :segment :analyzed_entity_id segment-id))))
+ (#'task.entity-check/check-entities!)
+ (testing "Both segment and dependent card should be re-analyzed"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :segment :analyzed_entity_id segment-id)))
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id card-id))))))))))
-;;; ### Table Metadata Update Tests
(deftest ^:sequential table-metadata-update-triggers-dependent-analysis-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Table metadata update triggers re-analysis of dependent cards"
+ (testing "Table metadata update marks immediate dependents stale"
(let [products-id (mt/id :products)
- orders-id (mt/id :orders)
+ orders (lib.metadata/table mp (mt/id :orders))
products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]
+ old-version models.analysis-finding/*current-analysis-finding-version*]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :table
- :to_entity_id products-id}]
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
(deps.findings/upsert-analysis! card)
(deps.findings/upsert-analysis! other-card)
- (testing "checking that initial analyses exist"
- (assert-has-analyses
- {:card {card-id old-version
- other-card-id old-version}}))
+ (assert-has-analyses {:card {card-id old-version other-card-id old-version}})
(let [table (t2/select-one :model/Table :id products-id)]
- (t2/with-transaction [_conn]
- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*}))
- (testing "card that depends on the updated table should be marked stale"
- (is (=? {:analysis_version old-version ; It hasn't been re-analyzed yet
- :stale true}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id))))
- (testing "card that does not depend on the updated table should not be marked stale"
- (is (=? {:analysis_version old-version
- :stale false}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id other-card-id))))))))))))
+ (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})
+ (testing "Card depending on updated table should be marked stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id card-id))))
+ (testing "Card not depending on updated table should not be stale"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))
(deftest ^:sequential table-metadata-update-works-with-no-analyses-test
(run-with-dependencies-setup!
@@ -1257,56 +946,35 @@
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :table
- :to_entity_id products-id}]
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
(let [table (t2/select-one :model/Table :id products-id)]
- (t2/with-transaction [_conn]
- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})
- (assert-has-analyses
- {:card {card-id nil}})))))))))
-
-;;; ### Field Metadata Update Tests
+ (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})
+ (assert-has-analyses {:card {card-id nil}}))))))))
(deftest ^:sequential field-metadata-update-triggers-dependent-analysis-test
(run-with-dependencies-setup!
(fn [mp]
- (testing "Field metadata update triggers re-analysis of cards depending on the field's table"
+ (testing "Field metadata update marks dependents of the field's table stale"
(let [products-id (mt/id :products)
- orders-id (mt/id :orders)
+ orders (lib.metadata/table mp (mt/id :orders))
products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)
- old-version models.analysis-finding/*current-analysis-finding-version*
- new-version (inc models.analysis-finding/*current-analysis-finding-version*)]
+ old-version models.analysis-finding/*current-analysis-finding-version*]
(mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}
:model/Card {other-card-id :id :as other-card} {:dataset_query (lib/query mp orders)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :table
- :to_entity_id products-id}]
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
(deps.findings/upsert-analysis! card)
(deps.findings/upsert-analysis! other-card)
- (testing "checking that initial analyses exist"
- (assert-has-analyses
- {:card {card-id old-version
- other-card-id old-version}}))
+ (assert-has-analyses {:card {card-id old-version other-card-id old-version}})
(let [field (t2/select-one :model/Field :id (mt/id :products :category))]
- (t2/with-transaction [_conn]
- (binding [models.analysis-finding/*current-analysis-finding-version* new-version]
- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*}))
- (testing "card that depends on the table whose field was updated should be marked stale"
- (is (=? {:analysis_version old-version ; It hasn't been re-analyzed yet
- :stale true}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id))))
- (testing "card that does not depend on the table whose field was updated should not be marked stale"
- (is (=? {:analysis_version old-version ; It hasn't been re-analyzed yet
- :stale false}
- (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id other-card-id))))))))))))
+ (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})
+ (testing "Card depending on the table whose field was updated should be stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id card-id))))
+ (testing "Card not depending on that table should not be stale"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))
(deftest ^:sequential field-metadata-update-works-with-no-analyses-test
(run-with-dependencies-setup!
@@ -1315,12 +983,8 @@
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :table
- :to_entity_id products-id}]
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
(let [field (t2/select-one :model/Field :id (mt/id :products :category))]
- (t2/with-transaction [_conn]
- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})
- (assert-has-analyses
- {:card {card-id nil}})))))))))
+ (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})
+ (assert-has-analyses {:card {card-id nil}}))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
index 72e737ff3a4..cefcd4ba51b 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
@@ -5,6 +5,7 @@
[metabase-enterprise.dependencies.analysis :as deps.analysis]
[metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -128,6 +129,13 @@
:analyzed_entity_type :card
:analyzed_entity_id [:in card-ids]))
+(defn- finding-stale?
+ "Returns the stale value for a specific entity's analysis finding, or nil if no finding exists."
+ [entity-type entity-id]
+ (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type entity-type
+ :analyzed_entity_id entity-id))
+
(deftest ^:sequential analyze-batch-picks-up-missing-analyses-test
(testing "analyze-batch! picks up entities with no pre-existing AnalysisFinding"
(backfill-all-entity-analyses!)
@@ -197,3 +205,75 @@
(is (= {grandparent-id false, parent-id true, child-id true}
(stale-map [grandparent-id parent-id child-id]))
"grandparent should NOT be stale, parent and child should be stale (transitive)")))))))))
+
+(deftest ^:sequential wave-propagation-via-entity-check-test
+ (testing "analyze-and-propagate! marks immediate dependents stale, and the job loop drains them in waves"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-premium-features #{:dependencies}
+ (lib-be/with-metadata-provider-cache
+ (mt/with-model-cleanup [:model/AnalysisFinding]
+ (mt/with-temp [:model/Card {gp-id :id :as gp-card} {:dataset_query (lib/query mp products)}
+ :model/Card {p-id :id :as p-card} {:dataset_query (lib/query mp (lib.metadata/card mp gp-id))}
+ :model/Card {c-id :id :as c-card} {:dataset_query (lib/query mp (lib.metadata/card mp p-id))}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id p-id
+ :to_entity_type :card :to_entity_id gp-id}
+ :model/Dependency _ {:from_entity_type :card :from_entity_id c-id
+ :to_entity_type :card :to_entity_id p-id}]
+ ;; Analyze all three so they have findings
+ (run! deps.findings/upsert-analysis! [gp-card p-card c-card])
+ (is (= {gp-id false, p-id false, c-id false}
+ (stale-map [gp-id p-id c-id]))
+ "all should start as not stale")
+ ;; Mark only the immediate dependents of grandparent (just parent, not child)
+ (deps.findings/mark-immediate-dependents-stale! :card gp-id)
+ (is (= {gp-id false, p-id true, c-id false}
+ (stale-map [gp-id p-id c-id]))
+ "only parent should be stale (immediate), child should NOT be stale yet")
+ ;; Run the entity-check job loop — it should analyze parent, which marks child stale,
+ ;; then analyze child in the next wave
+ (#'task.entity-check/check-entities!)
+ (is (= {gp-id false, p-id false, c-id false}
+ (stale-map [gp-id p-id c-id]))
+ "after check-entities!, all should be re-analyzed (not stale) — wave propagation worked"))))))))
+
+(deftest ^:sequential wave-propagation-through-non-analyzable-entity-test
+ (testing "Wave propagation skips non-analyzable intermediaries (e.g., tables) and reaches cards beyond them"
+ (backfill-all-entity-analyses!)
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))
+ products-id (mt/id :products)]
+ (mt/with-premium-features #{:dependencies}
+ (lib-be/with-metadata-provider-cache
+ (mt/with-model-cleanup [:model/AnalysisFinding]
+ ;; Chain: transform → (depends on) table ← (depends on) card
+ ;; Table is not analyzable, but card should still be reached by the wave
+ (mt/with-temp [:model/Transform {transform-id :id :as transform}
+ {:source {:type :query :query (lib/query mp products)}
+ :name "test_wave_transform"
+ :target {:schema "public" :name "wave_test_table" :type :table}}
+ :model/Card {card-id :id :as card}
+ {:dataset_query (lib/query mp products)}
+ ;; table depends on transform (downstream of transform)
+ :model/Dependency _ {:from_entity_type :table :from_entity_id products-id
+ :to_entity_type :transform :to_entity_id transform-id}
+ ;; card depends on the same table
+ :model/Dependency _ {:from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id products-id}]
+ ;; Analyze the transform and card (tables aren't analyzable)
+ (deps.findings/upsert-analysis! transform)
+ (deps.findings/upsert-analysis! card)
+ (is (false? (finding-stale? :card card-id))
+ "card should start as not stale")
+ ;; Mark only immediate dependents of transform stale
+ ;; The immediate dependent is the table, which is NOT analyzable
+ ;; The card is two hops away: transform → table → card
+ (deps.findings/mark-immediate-dependents-stale! :transform transform-id)
+ ;; The look-through should have reached the card through the non-analyzable table
+ (is (true? (finding-stale? :card card-id))
+ "card should be stale — look-through reached it via non-analyzable table")
+ ;; Run entity-check to drain stale entities
+ (#'task.entity-check/check-entities!)
+ (is (false? (finding-stale? :card card-id))
+ "card should be re-analyzed after entity-check"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
index f4c09abbe98..f7456a0009e 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
@@ -3,6 +3,7 @@
[clojure.set :as set]
[clojure.test :refer [deftest is testing]]
[metabase-enterprise.dependencies.models.dependency :as deps.graph]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.graph.core :as graph]
[metabase.lib.core :as lib]
[metabase.queries.models.card :as card]
@@ -48,8 +49,9 @@
(mt/dataset test-data
(mt/with-temp [:model/User user {:email "[email protected]"}]
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [card1 (card/create-card! (basic-orders) user)]
+ (deps.test/synchronously-run-backfill!)
(is (integer? (:id card1)))
(testing "when creating a new card"
(is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))}
@@ -57,6 +59,7 @@
(testing "that depends on another card"
(let [card2 (card/create-card! (wrap-card card1) user)]
+ (deps.test/synchronously-run-backfill!)
(is (=? #{(depends-on-> :card (:id card2) :card (:id card1))}
(upstream-of :card (:id card2))))
(testing "but that doesn't affect the upstream deps of the inner card"
@@ -71,6 +74,7 @@
{:joins [{:alias "Products"
:source-table (mt/id :products)
:condition [:= $id &Products.$products.id]}]})}})
+ (deps.test/synchronously-run-backfill!)
(is (=? #{(depends-on-> :card (:id card1) :table (mt/id :orders))
(depends-on-> :card (:id card1) :table (mt/id :products))}
(upstream-of :card (:id card1)))))
@@ -78,6 +82,7 @@
(card/update-card! {:card-before-update (t2/select-one :model/Card :id (:id card1))
:card-updates
{:dataset_query (mt/mbql-query products)}})
+ (deps.test/synchronously-run-backfill!)
(is (=? #{(depends-on-> :card (:id card1) :table (mt/id :products))}
(upstream-of :card (:id card1)))))))))))))
@@ -86,10 +91,11 @@
(mt/dataset test-data
(mt/with-temp [:model/User user {:email "[email protected]"}]
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
{id2 :id :as card2} (card/create-card! (wrap-card card1) user)
{id3 :id :as card3} (card/create-card! (wrap-card card2) user)]
+ (deps.test/synchronously-run-backfill!)
(testing "raw deps are recorded correctly"
(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
(is (=? #{(depends-on-> :card id2 :card id1)} (upstream-of :card id2)))
@@ -124,11 +130,12 @@
(mt/dataset test-data
(mt/with-temp [:model/User user {:email "[email protected]"}]
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [mp (mt/metadata-provider)
{id1 :id :as card1} (card/create-card! (sql-card mp "SELECT * FROM orders;") user)
{id2 :id :as card2} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id1 "}}")) user)
{id3 :id :as card3} (card/create-card! (sql-card mp (str "SELECT * FROM {{#" id2 "}}")) user)]
+ (deps.test/synchronously-run-backfill!)
(testing "raw deps are recorded correctly"
(is (=? #{(depends-on-> :card id1 :table (mt/id :orders))} (upstream-of :card id1)))
(is (=? #{(depends-on-> :card id2 :table (mt/id :orders))
@@ -154,7 +161,7 @@
(mt/dataset test-data
(mt/with-temp [:model/User user {:email "[email protected]"}]
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [metric-card (card/create-card! {:name "Test Metric"
:database_id (mt/id)
:table_id (mt/id :orders)
@@ -173,102 +180,95 @@
{:aggregation [[:metric (:id metric-card)]]})
:visualization_settings {}}
user)]
+ (deps.test/synchronously-run-backfill!)
(testing "raw deps are recorded correctly for question using metric"
(is (=? #{(depends-on-> :card (:id question-card) :table (mt/id :orders))
(depends-on-> :card (:id question-card) :card (:id metric-card))}
(upstream-of :card (:id question-card))))))))))))
-(deftest dependency-analysis-version-test
- (testing "dependency_analysis_version is updated when entities are created or updated"
- (mt/with-empty-h2-app-db!
+(deftest dependency-status-marked-stale-test
+ (testing "dependency_status is marked stale when entities are created or updated"
+ (mt/with-model-cleanup [:model/DependencyStatus]
(testing "cards"
(mt/with-model-cleanup [:model/Dependency]
- (mt/with-temp [:model/Card card {:dependency_analysis_version 0}]
- (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Card (:id card))))
+ (mt/with-temp [:model/Card card {}]
(mt/with-premium-features #{:dependencies}
(card/update-card! {:card-before-update card
:card-updates {:dataset_query (mt/mbql-query orders)}}))
- (is (= deps.graph/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Card (:id card)))))))
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true)))))
(testing "transforms create"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/Transform transform]
- (is (= deps.graph/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))
+ (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))
(testing "transforms update"
(mt/with-premium-features #{}
(mt/with-temp [:model/Transform transform]
- (is (zero? (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform))))
(mt/with-premium-features #{:dependencies}
(t2/update! :model/Transform (:id transform) {:source {:type "query" :query (mt/mbql-query products)}}))
- (is (= deps.graph/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Transform (:id transform)))))))
+ (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))
(testing "snippets create"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/NativeQuerySnippet snippet]
- (is (= deps.graph/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet)))))))
+ (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true)))))
(testing "snippets update"
(mt/with-premium-features #{}
(mt/with-temp [:model/NativeQuerySnippet snippet]
- (is (zero? (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet))))
(mt/with-premium-features #{:dependencies}
(t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))
- (is (= deps.graph/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/NativeQuerySnippet (:id snippet))))))))))
+ (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true))))))))
-(deftest dependency-analysis-version-create-card-test
- (testing "dependency_analysis_version is updated when a card is created"
+(deftest dependency-status-marked-stale-on-card-create-test
+ (testing "dependency_status is marked stale when a card is created"
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user]
(let [card (card/create-card! (basic-orders) user)]
- (is (= deps.graph/current-dependency-analysis-version
- (t2/select-one-fn :dependency_analysis_version :model/Card :id (:id card))))))))))
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true))))))))
(deftest filtered-graph-dependencies-test
(testing "filtered-graph-dependencies respects filter clause"
- (mt/with-empty-h2-app-db!
- (mt/with-temp [:model/User user {:email "[email protected]"}]
- (mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
- (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
- {id2 :id :as card2} (card/create-card! (wrap-card card1) user)
- {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]
- (testing "without filter, returns all dependencies"
- (let [graph (deps.graph/graph-dependencies)
- deps (graph/transitive graph [[:card id3]])]
- (is (= #{[:card id2] [:card id1] [:table (mt/id :orders)]}
- (set deps)))))
- (testing "with filter excluding card1, omits card1 and its dependencies"
- (let [filter-fn (fn [entity-type-field entity-id-field]
- [:and
- [:= entity-type-field "card"]
- [:in entity-id-field [id2 id3]]])
- graph (deps.graph/filtered-graph-dependencies filter-fn)
- deps (graph/transitive graph [[:card id3]])]
- (is (= #{[:card id2]}
- (set deps))
- "Should only include card2, not card1 or table")))
- (testing "with filter excluding card2, breaks the chain"
- (let [filter-fn (fn [entity-type-field entity-id-field]
- [:and
- [:= entity-type-field "card"]
- [:in entity-id-field [id1 id3]]])
- graph (deps.graph/filtered-graph-dependencies filter-fn)
- deps (graph/transitive graph [[:card id3]])]
- (is (= #{}
- (set deps))
- "Should be empty, chain is broken at card2"))))))))))
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
+ (let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
+ {id2 :id :as card2} (card/create-card! (wrap-card card1) user)
+ {id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]
+ (deps.test/synchronously-run-backfill!)
+ (testing "without filter, returns all dependencies"
+ (let [graph (deps.graph/graph-dependencies)
+ deps (graph/transitive graph [[:card id3]])]
+ (is (= #{[:card id2] [:card id1] [:table (mt/id :orders)]}
+ (set deps)))))
+ (testing "with filter excluding card1, omits card1 and its dependencies"
+ (let [filter-fn (fn [entity-type-field entity-id-field]
+ [:and
+ [:= entity-type-field "card"]
+ [:in entity-id-field [id2 id3]]])
+ graph (deps.graph/filtered-graph-dependencies filter-fn)
+ deps (graph/transitive graph [[:card id3]])]
+ (is (= #{[:card id2]}
+ (set deps))
+ "Should only include card2, not card1 or table")))
+ (testing "with filter excluding card2, breaks the chain"
+ (let [filter-fn (fn [entity-type-field entity-id-field]
+ [:and
+ [:= entity-type-field "card"]
+ [:in entity-id-field [id1 id3]]])
+ graph (deps.graph/filtered-graph-dependencies filter-fn)
+ deps (graph/transitive graph [[:card id3]])]
+ (is (= #{}
+ (set deps))
+ "Should be empty, chain is broken at card2")))))))))
(deftest filtered-graph-dependents-test
(testing "filtered-graph-dependents respects filter clause"
(mt/with-temp [:model/User user {:email "[email protected]"}]
(mt/with-premium-features #{:dependencies}
- (mt/with-model-cleanup [:model/Card :model/Dependency]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(let [{id1 :id :as card1} (card/create-card! (basic-orders) user)
{id2 :id :as card2} (card/create-card! (wrap-card card1) user)
{id3 :id :as _card3} (card/create-card! (wrap-card card2) user)]
+ (deps.test/synchronously-run-backfill!)
(testing "without filter, returns all dependents"
(let [graph (deps.graph/graph-dependents)
deps (graph/transitive graph [[:card id1]])]
@@ -308,6 +308,7 @@
:to_entity_type :table
:to_entity_id (:id table1)})
(deps.graph/swap-dependency! :card (:id card1) [:table (:id table1)] [:table (:id table2)])
+ (deps.test/synchronously-run-backfill!)
(is (not (t2/exists? :model/Dependency
:from_entity_type :card
:from_entity_id (:id card1)
@@ -333,6 +334,7 @@
:to_entity_id (:id table2)}])
;; This would fail with duplicate key error before the fix
(deps.graph/swap-dependency! :card (:id card2) [:table (:id table1)] [:table (:id table2)])
+ (deps.test/synchronously-run-backfill!)
(is (not (t2/exists? :model/Dependency
:from_entity_type :card
:from_entity_id (:id card2)
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
index 50f46f7afbb..4d06f647aae 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
@@ -3,8 +3,11 @@
[clojure.test :refer :all]
[environ.core :as env]
[java-time.api :as t]
+ [metabase-enterprise.dependencies.calculation :as deps.calculation]
[metabase-enterprise.dependencies.models.dependency :as dependencies.model]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
[metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.events.core :as events]
[metabase.task.core :as task]
[metabase.test :as mt]
@@ -17,59 +20,76 @@
(defn- backfill-all-existing-entities!
[]
- (mt/with-premium-features #{:dependencies}
- (while (#'dependencies.backfill/backfill-dependencies!))))
+ (deps.test/synchronously-run-backfill!))
(defn- backfill-dependencies-single-trigger!
[]
(mt/with-premium-features #{:dependencies}
(#'dependencies.backfill/backfill-dependencies!)))
+(defn- mark-stale!
+ "Mark an entity as stale in dependency_status for testing."
+ [entity-type entity-id]
+ (deps.dependency-status/mark-stale! entity-type [entity-id]))
+
+(defn- assert-processed
+ "Assert that an entity has been processed (not stale, current version)."
+ [entity-type entity-id]
+ (is (t2/exists? :model/DependencyStatus
+ :entity_type entity-type
+ :entity_id entity-id
+ :stale false
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version)
+ (str "Expected " (name entity-type) " " entity-id " to be processed")))
+
(deftest ^:sequential backfill-dependency-analysis-test
- (testing "Test that the backfill job correctly updates the dependency_analysis_version"
+ (testing "Test that the backfill job correctly processes stale entities"
(backfill-all-existing-entities!)
(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
(let [query (mt/mbql-query orders)]
(mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}
- :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}
- :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]
- (let [card-count (fn [& args]
- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
- current-version dependencies.model/current-dependency-analysis-version]
- (is (= 3 (card-count)))
+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
+ :model/Card {card2-id :id} {:dataset_query query}
+ :model/Card {card3-id :id} {:dataset_query query}]
+ ;; Mark all three cards as stale
+ (mark-stale! :card card1-id)
+ (mark-stale! :card card2-id)
+ (mark-stale! :card card3-id)
+ (let [stale-count (fn []
+ (t2/count :model/DependencyStatus
+ :entity_type :card
+ :entity_id [:in [card1-id card2-id card3-id]]
+ :stale true))]
+ (is (= 3 (stale-count)))
;; first run, should process 2 cards
(is (true? (backfill-dependencies-single-trigger!)))
- (is (= 2 (card-count :dependency_analysis_version current-version)))
+ (is (= 1 (stale-count)))
;; second run, should process the last card
(is (false? (backfill-dependencies-single-trigger!)))
- (is (= 3 (card-count :dependency_analysis_version current-version)))
+ (is (= 0 (stale-count)))
;; third run, should not process anything
(is (false? (backfill-dependencies-single-trigger!)))
- (is (= 3 (card-count :dependency_analysis_version current-version))))))))))
+ (is (= 0 (stale-count))))))))))
(deftest ^:sequential backfill-transform-test
(testing "Test that transforms are correctly backfilled"
(backfill-all-existing-entities!)
(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
(mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)
- :dependency_analysis_version 0}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}
:model/Transform {transform-id :id} {:name "Test Transform"
:source {:type "query"
:query (mt/mbql-query nil
{:source-table (str "card__" card-id)})}
- :target {:type "table" :name (mt/random-name)}
- :dependency_analysis_version 0}]
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
- (is (t2/exists? :model/Transform :id transform-id :dependency_analysis_version 0))
+ :target {:type "table" :name (mt/random-name)}}]
+ ;; Mark both as stale
+ (mark-stale! :card card-id)
+ (mark-stale! :transform transform-id)
(is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id)))
(is (false? (t2/exists? :model/Dependency :from_entity_type :transform :from_entity_id transform-id)))
(backfill-dependencies-single-trigger!)
- (is (t2/exists? :model/Card :id card-id
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
- (is (t2/exists? :model/Transform :id transform-id
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (assert-processed :card card-id)
+ (assert-processed :transform transform-id)
(is (t2/exists? :model/Dependency
:from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :orders)))
@@ -83,8 +103,7 @@
(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "2")]
(mt/with-premium-features #{}
(mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "my_snippet"
- :content "SELECT 1"
- :dependency_analysis_version 0}
+ :content "SELECT 1"}
:model/Card {card-id :id} {:dataset_query {:database (mt/id)
:type :native
:native {:query "SELECT * FROM {{my_snippet}}"
@@ -95,18 +114,16 @@
:display-name "my_snippet"
:type :snippet
:snippet-name "my_snippet"
- :snippet-id snippet-id}}}}
- :dependency_analysis_version 0}]
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
- (is (t2/exists? :model/NativeQuerySnippet :id snippet-id :dependency_analysis_version 0))
+ :snippet-id snippet-id}}}}}]
+ ;; Mark both as stale
+ (mark-stale! :card card-id)
+ (mark-stale! :snippet snippet-id)
(is (false? (t2/exists? :model/Dependency
:from_entity_type :card :from_entity_id card-id
:to_entity_type :snippet :to_entity_id snippet-id)))
(backfill-dependencies-single-trigger!)
- (is (t2/exists? :model/Card :id card-id
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
- (is (t2/exists? :model/NativeQuerySnippet :id snippet-id
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (assert-processed :card card-id)
+ (assert-processed :snippet snippet-id)
(is (t2/exists? :model/Dependency
:from_entity_type :card :from_entity_id card-id
:to_entity_type :snippet :to_entity_id snippet-id)))))))
@@ -116,19 +133,16 @@
(backfill-all-existing-entities!)
(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "1")]
(mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)
- :dependency_analysis_version 0}]
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (mark-stale! :card card-id)
(is (false? (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :orders))))
;; First run
(backfill-dependencies-single-trigger!)
- (is (t2/exists? :model/Card :id card-id
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (assert-processed :card card-id)
;; Second run - should not change anything
(backfill-dependencies-single-trigger!)
- (is (t2/exists? :model/Card :id card-id
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))
+ (assert-processed :card card-id)
(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :orders))))))))
@@ -149,132 +163,220 @@
:mb-dependency-backfill-batch-size "1"
:mb-dependency-backfill-delay-minutes "0"
:mb-dependency-backfill-variance-minutes "0")]
- (let [card-data {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+ (let [card-data {:dataset_query (mt/mbql-query orders)}]
(mt/test-helpers-set-global-values!
(mt/with-premium-features #{}
(mt/with-temp [:model/Card {card1-id :id} card-data
:model/Card {card2-id :id} card-data
:model/Card {card3-id :id} card-data]
- (let [card-count (fn [& args]
- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
- current-version dependencies.model/current-dependency-analysis-version]
- (is (= 3 (card-count :dependency_analysis_version 0)))
+ ;; Mark all as stale
+ (mark-stale! :card card1-id)
+ (mark-stale! :card card2-id)
+ (mark-stale! :card card3-id)
+ (let [processed? (fn []
+ (= 3 (t2/count :model/DependencyStatus
+ :entity_type :card
+ :entity_id [:in [card1-id card2-id card3-id]]
+ :stale false
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version)))]
+ (is (not (processed?)))
(mt/with-premium-features #{:dependencies}
;; Initialize the task, which should schedule the first run
(task/init! ::dependencies.backfill/DependencyBackfill)
- (wait-for-condition #(= 3 (card-count :dependency_analysis_version current-version)) 10000)
- (is (= 3 (card-count :dependency_analysis_version current-version)))))))))))))
+ (wait-for-condition processed? 10000)
+ (is (processed?))))))))))))
+
+(deftest ^:sequential backfill-error-logging-test
+ (testing "When calculate-deps throws, the error is logged and the entity remains stale"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (mark-stale! :card card-id)
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-delay-minutes "0"
+ :mb-dependency-backfill-variance-minutes "0")
+ deps.calculation/calculate-deps
+ (fn [_ _] (throw (ex-info "Simulated error" {})))]
+ (backfill-dependencies-single-trigger!))
+ ;; Entity should still be stale — not processed, not lost
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
+ ;; No dependencies should have been created
+ (is (empty? (t2/select :model/Dependency :from_entity_type :card :from_entity_id card-id)))
+ ;; Failure recorded in table
+ (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus
+ :entity_type :card :entity_id card-id)))))))
+
+(deftest ^:sequential backfill-partial-batch-failure-test
+ (testing "A failure on one entity doesn't prevent other entities from being processed"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {good-card-id :id} {:dataset_query (mt/mbql-query orders)}
+ :model/Card {bad-card-id :id} {:dataset_query (mt/mbql-query products)}]
+ (mark-stale! :card good-card-id)
+ (mark-stale! :card bad-card-id)
+ (let [original-calculate-deps deps.calculation/calculate-deps]
+ (with-redefs [env/env (assoc env/env
+ :mb-dependency-backfill-delay-minutes "0"
+ :mb-dependency-backfill-variance-minutes "0")
+ deps.calculation/calculate-deps
+ (fn [entity-type entity]
+ (if (= (:id entity) bad-card-id)
+ (throw (ex-info "Simulated error" {}))
+ (original-calculate-deps entity-type entity)))]
+ (backfill-dependencies-single-trigger!)))
+ ;; Good card should be processed successfully
+ (assert-processed :card good-card-id)
+ (is (seq (t2/select :model/Dependency :from_entity_type :card :from_entity_id good-card-id)))
+ ;; Bad card should remain stale with failure recorded
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id bad-card-id :stale true))
+ (is (= 1 (t2/select-one-fn :fail_count :model/DependencyStatus
+ :entity_type :card :entity_id bad-card-id)))))))
(deftest ^:sequential backfill-terminal-failure-test
(testing "Entities should be marked as terminally broken after MAX_RETRIES failures"
(backfill-all-existing-entities!)
(mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (mark-stale! :card card-id)
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- (let [update-attempts (volatile! 0)
+ (let [compute-attempts (volatile! 0)
failures (inc @#'dependencies.backfill/max-retries)]
(with-redefs [env/env (assoc env/env
:mb-dependency-backfill-delay-minutes "0"
:mb-dependency-backfill-variance-minutes "0")
- t2/update! (fn [model-kw id & args]
- (if (and (= model-kw :model/Card)
- (= id card-id)
- (< @update-attempts failures))
- (do
- (vswap! update-attempts inc)
- (throw (ex-info "Simulated DB error" {:id id})))
- (apply t2/update! model-kw id args)))]
+ ;; Make compute-deps-for-entity! throw
+ deps.calculation/calculate-deps
+ (fn [_ _]
+ (vswap! compute-attempts inc)
+ (throw (ex-info "Simulated computation error" {:card-id card-id})))]
;; fail MAX_RETRIES + 1 times
- (while (< @update-attempts failures)
+ (while (< @compute-attempts failures)
(backfill-dependencies-single-trigger!))))
- ;; verify card is not processed and is terminally broken
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ ;; verify card is still stale (not processed)
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
;; verify subsequent runs don't process it
(backfill-dependencies-single-trigger!)
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))))))
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))))))
(deftest ^:sequential backfill-delayed-retry-test
(testing "Failed entities should be retried after their delay period expires"
(backfill-all-existing-entities!)
- (let [current-version dependencies.model/current-dependency-analysis-version
- update-attempts (volatile! 0)]
+ (let [compute-attempts (volatile! 0)]
(mt/with-premium-features #{}
- (mt/with-temp [:model/Card {card-id :id} {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (mark-stale! :card card-id)
(with-redefs [env/env (assoc env/env
- :mb-dependency-backfill-delay-minutes "1") ; 1 minute delay for retries
- t2/update! (fn [model-kw id & args]
- (if (and (= model-kw :model/Card)
- (= id card-id)
- (zero? @update-attempts))
- (do
- (vswap! update-attempts inc)
- (throw (ex-info "Simulated DB error" {:id id})))
- (apply t2/update! model-kw id args)))]
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ :mb-dependency-backfill-delay-minutes "1")
+ deps.calculation/calculate-deps
+ (fn [_ _entity]
+ (if (zero? @compute-attempts)
+ (do
+ (vswap! compute-attempts inc)
+ (throw (ex-info "Simulated computation error" {:card-id card-id})))
+ ;; Return valid deps on subsequent attempts
+ {:table #{(mt/id :orders)}}))]
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
;; first failure - should be put into retry state
- ;; fail MAX_RETRIES + 1 times
- (while (zero? @update-attempts)
+ (while (zero? @compute-attempts)
(backfill-dependencies-single-trigger!))
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0)))
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- ;; advance time by less than retry delay - should NOT be processed
- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))
- (backfill-dependencies-single-trigger!))
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version 0))
+ ;; advance time by less than retry delay - should NOT be processed
+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 10 :seconds))
+ (backfill-dependencies-single-trigger!))
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- ;; advance time by more than retry delay - should be processed
- (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))
- (backfill-dependencies-single-trigger!))
- (is (t2/exists? :model/Card :id card-id :dependency_analysis_version current-version)))))))
+ ;; advance time by more than retry delay - should be processed
+ (mt/with-clock (t/plus (t/zoned-date-time) (t/duration 2 :minutes))
+ (backfill-dependencies-single-trigger!))
+ (assert-processed :card card-id)))))))
(deftest backfill-card-does-not-cause-revision-test
(testing "backfilling a card does not create a new revision or audit log entry"
(backfill-all-existing-entities!)
(mt/with-premium-features #{:dependencies :audit-app}
- (mt/with-temp [:model/Card {card-id :id
- dep-version :dependency_analysis_version}
- {:dependency_analysis_version 0, :dataset_query (mt/mbql-query orders)}]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ (mark-stale! :card card-id)
(let [revision-count-before (t2/count :model/Revision :model "Card" :model_id card-id)
deps-before (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]
- (is (= 0 dep-version))
(is (= 0 revision-count-before))
(is (= 0 deps-before))
(backfill-dependencies-single-trigger!)
(let [revision-count-after (t2/count :model/Revision :model "Card" :model_id card-id)
- dep-version-after (t2/select-one-fn :dependency_analysis_version :model/Card :id card-id)
deps-after (t2/count :model/Dependency :from_entity_type :card :from_entity_id card-id)]
(is (= 0 revision-count-after))
- (is (= dependencies.model/current-dependency-analysis-version dep-version-after))
+ (assert-processed :card card-id)
(is (= 1 deps-after))))))))
(deftest ^:sequential backfill-dependencies-on-serdes-load-test
- (testing "Test that serdes load correctly updates the dependency_analysis_version"
+ (testing "Serdes load triggers the backfill job (entities remain stale until job runs)"
(backfill-all-existing-entities!)
(let [query (mt/mbql-query orders)]
(mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}
- :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}
- :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]
- (let [card-count (fn [& args]
- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
- current-version dependencies.model/current-dependency-analysis-version]
- (events/publish-event! :event/serdes-load {})
- (is (= 3 (card-count :dependency_analysis_version current-version)))))))))
+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
+ :model/Card {card2-id :id} {:dataset_query query}
+ :model/Card {card3-id :id} {:dataset_query query}]
+ ;; Mark all as stale
+ (mark-stale! :card card1-id)
+ (mark-stale! :card card2-id)
+ (mark-stale! :card card3-id)
+ ;; serdes-load triggers the async job, not synchronous processing
+ (events/publish-event! :event/serdes-load {})
+ ;; entities are still stale (job hasn't run yet in this test)
+ (is (= 3 (t2/count :model/DependencyStatus
+ :entity_type :card
+ :entity_id [:in [card1-id card2-id card3-id]]
+ :stale true)))
+ ;; manually run the backfill to verify it processes them
+ (backfill-all-existing-entities!)
+ (is (= 3 (t2/count :model/DependencyStatus
+ :entity_type :card
+ :entity_id [:in [card1-id card2-id card3-id]]
+ :stale false
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))
(deftest ^:sequential backfill-dependencies-on-token-update-test
- (testing "Test that token update correctly updates the dependency_analysis_version"
+ (testing "Token update triggers the backfill job (entities remain stale until job runs)"
(backfill-all-existing-entities!)
(let [query (mt/mbql-query orders)]
(mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card1-id :id} {:dependency_analysis_version 0, :dataset_query query}
- :model/Card {card2-id :id} {:dependency_analysis_version 0, :dataset_query query}
- :model/Card {card3-id :id} {:dependency_analysis_version 0, :dataset_query query}]
- (let [card-count (fn [& args]
- (apply t2/count :model/Card :id [:in [card1-id card2-id card3-id]] args))
- current-version dependencies.model/current-dependency-analysis-version]
- (events/publish-event! :event/set-premium-embedding-token {})
- (is (= 3 (card-count :dependency_analysis_version current-version)))))))))
+ (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
+ :model/Card {card2-id :id} {:dataset_query query}
+ :model/Card {card3-id :id} {:dataset_query query}]
+ ;; Mark all as stale
+ (mark-stale! :card card1-id)
+ (mark-stale! :card card2-id)
+ (mark-stale! :card card3-id)
+ ;; token update triggers the async job, not synchronous processing
+ (events/publish-event! :event/set-premium-embedding-token {})
+ ;; entities are still stale (job hasn't run yet in this test)
+ (is (= 3 (t2/count :model/DependencyStatus
+ :entity_type :card
+ :entity_id [:in [card1-id card2-id card3-id]]
+ :stale true)))
+ ;; manually run the backfill to verify it processes them
+ (backfill-all-existing-entities!)
+ (is (= 3 (t2/count :model/DependencyStatus
+ :entity_type :card
+ :entity_id [:in [card1-id card2-id card3-id]]
+ :stale false
+ :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))
+
+(deftest ^:sequential backfill-version-outdated-test
+ (testing "Entities with outdated version in dependency_status get reprocessed"
+ (backfill-all-existing-entities!)
+ (mt/with-premium-features #{}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query orders)}]
+ ;; Create a dependency_status entry with an old version (not stale, but outdated)
+ (t2/insert! :model/DependencyStatus {:entity_type :card
+ :entity_id card-id
+ :dependency_analysis_version 0
+ :stale false})
+ (backfill-dependencies-single-trigger!)
+ (assert-processed :card card-id)
+ (is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
+ :to_entity_type :table :to_entity_id (mt/id :orders)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
index 29e04d9bbde..42950daa0b1 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
@@ -1,11 +1,21 @@
(ns metabase-enterprise.dependencies.test-util
(:require
[medley.core :as m]
+ [metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
[metabase.lib.convert :as lib.convert]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
- [metabase.lib.test-util.metadata-providers.mock :as providers.mock]))
+ [metabase.lib.test-util.metadata-providers.mock :as providers.mock]
+ [metabase.test :as mt]))
+
+(defn synchronously-run-backfill!
+ "Run the dependency backfill job synchronously, processing all stale/outdated entities
+ until there is nothing left to process. Use this in tests after operations that mark
+ entities stale (create/update events) and before asserting on dependency data."
+ []
+ (mt/with-premium-features #{:dependencies}
+ (while (#'dependencies.backfill/backfill-dependencies!))))
(defn mock-card [metadata-provider {:keys [id query details]}]
(merge {:lib/type :metadata/card
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
index e63f284e70e..d5c37e848b5 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.execute :as replacement.execute]
[metabase-enterprise.replacement.models.replacement-run :as replacement-run]
[metabase-enterprise.replacement.protocols :as replacement.protocols]
@@ -134,6 +135,7 @@
;; Populate dependencies via events
(doseq [card [old-model mbql-child-1 mbql-child-2 native-child grandchild grandchild-native]]
(events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"
{:source_entity_id old-id
@@ -199,6 +201,7 @@
:name "Child card"}]
(mt/with-model-cleanup [:model/ReplacementRun :model/Dependency]
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
;; Insert a fake active run to simulate one already running
(let [run (replacement-run/create-run! :card old-id :card new-id (mt/user->id :crowberto))]
(replacement-run/start-run! (:id run)))
@@ -230,6 +233,7 @@
:name "Child Card"}]
(mt/with-model-cleanup [:model/ReplacementRun :model/Dependency]
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"
{:source_entity_id old-id
:source_entity_type :card
@@ -262,6 +266,7 @@
:name "Child card"}]
(mt/with-model-cleanup [:model/ReplacementRun :model/Dependency]
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :crowberto)})
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :post 202 "ee/replacement/replace-source"
{:source_entity_id old-id
:source_entity_type :card
@@ -371,6 +376,7 @@
;; Populate dependencies via events
(doseq [card [model-card question-card]]
(events/publish-event! :event/card-create {:object card :user-id (mt/user->id :crowberto)}))
+ (deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :post 202
"ee/replacement/replace-model-with-transform"
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
index e52e6130f0a..9577079307e 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
@@ -3,6 +3,7 @@
(:require
[clojure.test :refer [deftest is testing]]
[metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.protocols :as replacement.protocols]
[metabase-enterprise.replacement.runner :as replacement.runner]
[metabase-enterprise.replacement.source-swap :as replacement.source-swap]
@@ -196,10 +197,11 @@
:dataset_query (lib/query mp (lib.metadata/card mp old-id))
:type :question
:name "Child Card"}]
- (mt/with-model-cleanup [:model/Dependency]
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
;; populate dependencies
(events/publish-event! :event/card-create {:object old-card :user-id (mt/user->id :rasta)})
(events/publish-event! :event/card-create {:object child-card :user-id (mt/user->id :rasta)})
+ (deps.test/synchronously-run-backfill!)
(testing "child card initially points to old model"
(is (= old-id (get-in (t2/select-one-fn :dataset_query :model/Card :id child-id)
@@ -214,6 +216,7 @@
(start-run! [_])
(succeed-run! [_])
(fail-run! [_ _]))]
+ #_{:clj-kondo/ignore [:unresolved-var]}
(replacement.runner/run-swap-source! [:card old-id] [:card new-id] progress)
(testing "child card's source-card is updated to new model"
@@ -254,9 +257,10 @@
:dataset_query (lib/query mp (lib.metadata/card mp old-id))
:type :question
:name "Child 2"}]
- (mt/with-model-cleanup [:model/Dependency]
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
(doseq [card [old-card child-1 child-2]]
(events/publish-event! :event/card-create {:object card :user-id (mt/user->id :rasta)}))
+ (deps.test/synchronously-run-backfill!)
(let [original-swap! replacement.source-swap/swap-source!]
(with-redefs [replacement.source-swap/swap-source!
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
index c66e04d9c1f..6ddddbb7fab 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
@@ -3,6 +3,7 @@
[clojure.test :refer [deftest is testing]]
[medley.core :as m]
[metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.field-refs :as replacement.field-refs]
[metabase-enterprise.replacement.source-swap :as replacement.source-swap]
[metabase.events.core :as events]
@@ -323,6 +324,7 @@
;; Create a dependency: card -> orders table
(events/publish-event! :event/card-create {:object (t2/select-one :model/Card card-id)
:user-id (mt/user->id :rasta)})
+ (deps.test/synchronously-run-backfill!)
(is (t2/exists? :model/Dependency
:from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :orders))
@@ -352,6 +354,7 @@
(mt/with-model-cleanup [:model/Dependency]
(events/publish-event! :event/segment-create {:object (t2/select-one :model/Segment segment-id)
:user-id (mt/user->id :rasta)})
+ (deps.test/synchronously-run-backfill!)
(is (t2/exists? :model/Dependency
:from_entity_type :segment :from_entity_id segment-id
:to_entity_type :table :to_entity_id (mt/id :orders)))
@@ -381,6 +384,7 @@
(mt/with-model-cleanup [:model/Dependency]
(events/publish-event! :event/measure-create {:object (t2/select-one :model/Measure measure-id)
:user-id (mt/user->id :rasta)})
+ (deps.test/synchronously-run-backfill!)
(is (t2/exists? :model/Dependency
:from_entity_type :measure :from_entity_id measure-id
:to_entity_type :table :to_entity_id (mt/id :orders)))
@@ -409,6 +413,7 @@
(mt/with-model-cleanup [:model/Dependency]
(events/publish-event! :event/transform-create {:object (t2/select-one :model/Transform transform-id)
:user-id (mt/user->id :rasta)})
+ (deps.test/synchronously-run-backfill!)
(is (t2/exists? :model/Dependency
:from_entity_type :transform :from_entity_id transform-id
:to_entity_type :table :to_entity_id (mt/id :orders)))
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
index a066305704a..59f9ad7d421 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer [deftest is testing]]
[metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.usages :as replacement.usages]
[metabase.graph.core :as graph]
[metabase.lib.core :as lib]
@@ -78,6 +79,7 @@
(mt/with-model-cleanup [:model/Card :model/Dependency]
(let [source-card (card/create-card! (card-with-query "Source card" :products) user)
child-card (card/create-card! (card-sourced-from "Child card" source-card) user)
+ _ (deps.test/synchronously-run-backfill!)
found-usages (replacement.usages/transitive-usages [:card (:id source-card)])]
(is (some #(= [:card (:id child-card)] %) found-usages)
"Child card should appear in usages"))))))))
@@ -91,6 +93,7 @@
(let [source-card (card/create-card! (card-with-query "Source card" :products) user)
child-card (card/create-card! (card-sourced-from "Child card" source-card) user)
grandchild-card (card/create-card! (card-sourced-from "Grandchild card" child-card) user)
+ _ (deps.test/synchronously-run-backfill!)
found-usages (replacement.usages/transitive-usages [:card (:id source-card)])]
(is (some #(= [:card (:id child-card)] %) found-usages)
"Child card should appear in usages")
@@ -104,6 +107,7 @@
(let [user (mt/fetch-user :rasta)]
(mt/with-model-cleanup [:model/Card :model/Dependency]
(let [lonely-card (card/create-card! (card-with-query "Lonely card" :products) user)
+ _ (deps.test/synchronously-run-backfill!)
found-usages (replacement.usages/transitive-usages [:card (:id lonely-card)])]
(is (empty? found-usages)
"Card with no dependents should have empty usages"))))))))
@@ -115,6 +119,7 @@
(let [user (mt/fetch-user :rasta)]
(mt/with-model-cleanup [:model/Card :model/Dependency]
(let [card-on-table (card/create-card! (card-with-query "Card on products" :products) user)
+ _ (deps.test/synchronously-run-backfill!)
found-usages (replacement.usages/transitive-usages [:table (mt/id :products)])]
(is (some #(= [:card (:id card-on-table)] %) found-usages)
"Card using table should appear in usages"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.clj
index eb86588f4cf..11f82ca0059 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/api/gtap_test.clj
@@ -23,8 +23,7 @@
:card_id true
:table_id true
:group_id true
- :attribute_remappings {:foo 1}
- :dependency_analysis_version 0})
+ :attribute_remappings {:foo 1}})
(defmacro ^:private with-gtap-cleanup!
"Invokes `body` ensuring any `Sandbox` created will be removed afterward. Leaving behind a GTAP can
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index 24fb05e57d3..69f533a7a05 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -52,6 +52,7 @@
:model/DataPermissions
:model/DatabaseRouter
:model/Dependency
+ :model/DependencyStatus
:model/DocumentBookmark
:model/CollectionPermissionGraphRevision
:model/DashboardCardSeries
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj
index ace7a1f4c2c..cab38ac77d1 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/test_util.clj
@@ -3,6 +3,7 @@
[clojure.set :as set]
[clojure.string :as str]
[clojure.test :refer :all]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.workspaces.common :as ws.common]
[metabase-enterprise.workspaces.dag-abstract :as dag-abstract]
[metabase-enterprise.workspaces.execute :as ws.execute]
@@ -315,6 +316,7 @@
;; Wait for workspace DB initialization and grant all inputs by default.
;; Test tables are metadata-only (no physical DB tables), so the isolation layer's GRANT
;; fails silently and `access_granted` stays false — we force-grant to avoid that.
+ _ (deps.test/synchronously-run-backfill!)
_ (when (and ws-id (not (:skip-init workspace)))
(ws-done! ws-id)
(force-grant-all-inputs! ws-id))]
diff --git a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj
index b4a20c1b746..25e77784aab 100644
--- a/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/workspaces/validation_test.clj
@@ -1,6 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.workspaces.validation-test
(:require
[clojure.test :refer :all]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.workspaces.impl :as ws.impl]
[metabase-enterprise.workspaces.test-util :as ws.tu]
[metabase-enterprise.workspaces.validation :as ws.validation]
@@ -90,6 +91,7 @@
:database (mt/id)
:schema "public"
:name "external_output"}}]
+ (deps.test/synchronously-run-backfill!)
;; Create a workspace transform that targets the same table
(let [{ref-id :ref_id} (mt/user-http-request :crowberto :post 200
(str "ee/workspace/" ws-id "/transform")
diff --git a/resources/migrations/060/20260402_dependency_status.yaml b/resources/migrations/060/20260402_dependency_status.yaml
new file mode 100644
index 00000000000..bcc59af98ca
--- /dev/null
+++ b/resources/migrations/060/20260402_dependency_status.yaml
@@ -0,0 +1,458 @@
+databaseChangeLog:
+
+ # 1. Create the dependency_status table
+ - changeSet:
+ id: v60.2026-04-02T00:00:00
+ author: ericnormand
+ comment: Create dependency_status table for async dependency calculation tracking
+ changes:
+ - createTable:
+ tableName: dependency_status
+ remarks: Tracks dependency calculation status for entities, analogous to analysis_finding
+ columns:
+ - column:
+ name: id
+ type: int
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: entity_type
+ type: varchar(254)
+ remarks: The type of the entity (card, transform, snippet, etc.)
+ constraints:
+ nullable: false
+ - column:
+ name: entity_id
+ type: int
+ remarks: The ID of the entity
+ constraints:
+ nullable: false
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ remarks: Version of the dependency analysis that was performed
+ constraints:
+ nullable: false
+ - column:
+ name: stale
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the entity needs dependency recalculation
+ constraints:
+ nullable: false
+ - column:
+ name: fail_count
+ type: int
+ defaultValueNumeric: 0
+ remarks: Number of consecutive failed dependency calculation attempts
+ constraints:
+ nullable: false
+ - column:
+ name: next_retry_at
+ type: ${timestamp_type}
+ remarks: Earliest time at which a failed entity can be retried (null means immediately processable)
+ - column:
+ name: terminal
+ type: ${boolean.type}
+ defaultValueBoolean: false
+ remarks: True when the entity has exceeded max retries and should not be retried until modified
+ constraints:
+ nullable: false
+ rollback:
+ - dropTable:
+ tableName: dependency_status
+
+ # 2. Unique constraint on (entity_type, entity_id)
+ - changeSet:
+ id: v60.2026-04-02T00:00:01
+ author: ericnormand
+ comment: Unique constraint to prevent duplicate dependency_status entries
+ changes:
+ - addUniqueConstraint:
+ tableName: dependency_status
+ columnNames: entity_type, entity_id
+ constraintName: idx_unique_dependency_status
+ rollback:
+ - dropUniqueConstraint:
+ tableName: dependency_status
+ constraintName: idx_unique_dependency_status
+
+ # 3. Index on stale column (partial on postgres, full on others)
+ - changeSet:
+ id: v60.2026-04-02T00:00:02
+ author: ericnormand
+ comment: Add partial index on dependency_status.stale for efficient polling
+ changes:
+ - sql:
+ dbms: postgresql
+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale) WHERE stale = true
+ - sql:
+ dbms: mysql,mariadb,h2
+ sql: CREATE INDEX idx_dependency_status_stale ON dependency_status(stale)
+ rollback:
+ - dropIndex:
+ tableName: dependency_status
+ indexName: idx_dependency_status_stale
+
+ # 4-11. Migrate data from entity tables to dependency_status
+ # All entities are migrated. Entities with version > 0 (already analyzed) get stale=false.
+ # Entities with version = 0 (never analyzed) get stale=true so the backfill picks them up.
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:03
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from report_card to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'card', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM report_card
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'card'
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:04
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from transform to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'transform', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM transform
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'transform'
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:05
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from native_query_snippet to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'snippet', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM native_query_snippet
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'snippet'
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:06
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from report_dashboard to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'dashboard', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM report_dashboard
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'dashboard'
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:07
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from document to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'document', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM document
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'document'
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:08
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from sandboxes to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'sandbox', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM sandboxes
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'sandbox'
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:09
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from segment to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'segment', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM segment
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'segment'
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:10
+ author: ericnormand
+ comment: Migrate dependency_analysis_version from measure to dependency_status
+ changes:
+ - sql:
+ sql: >-
+ INSERT INTO dependency_status (entity_type, entity_id, dependency_analysis_version, stale)
+ SELECT 'measure', id, dependency_analysis_version, CASE WHEN dependency_analysis_version = 0 THEN true ELSE false END FROM measure
+ rollback:
+ - sql:
+ sql: DELETE FROM dependency_status WHERE entity_type = 'measure'
+
+ # 12-17. Drop indexes on dependency_analysis_version (for tables that have them)
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:11
+ author: ericnormand
+ comment: Drop dependency_analysis_version index from report_card
+ changes:
+ - dropIndex:
+ tableName: report_card
+ indexName: idx_report_card_dependency_analysis_version
+ rollback:
+ - createIndex:
+ tableName: report_card
+ indexName: idx_report_card_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:12
+ author: ericnormand
+ comment: Drop dependency_analysis_version index from transform
+ changes:
+ - dropIndex:
+ tableName: transform
+ indexName: idx_transform_dependency_analysis_version
+ rollback:
+ - createIndex:
+ tableName: transform
+ indexName: idx_transform_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:13
+ author: ericnormand
+ comment: Drop dependency_analysis_version index from native_query_snippet
+ changes:
+ - dropIndex:
+ tableName: native_query_snippet
+ indexName: idx_native_query_snippet_dependency_analysis_version
+ rollback:
+ - createIndex:
+ tableName: native_query_snippet
+ indexName: idx_native_query_snippet_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:14
+ author: ericnormand
+ comment: Drop dependency_analysis_version index from report_dashboard
+ changes:
+ - dropIndex:
+ tableName: report_dashboard
+ indexName: idx_report_dashboard_dependency_analysis_version
+ rollback:
+ - createIndex:
+ tableName: report_dashboard
+ indexName: idx_report_dashboard_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:15
+ author: ericnormand
+ comment: Drop dependency_analysis_version index from document
+ changes:
+ - dropIndex:
+ tableName: document
+ indexName: idx_document_dependency_analysis_version
+ rollback:
+ - createIndex:
+ tableName: document
+ indexName: idx_document_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:16
+ author: ericnormand
+ comment: Drop dependency_analysis_version index from sandboxes
+ changes:
+ - dropIndex:
+ tableName: sandboxes
+ indexName: idx_sandboxes_dependency_analysis_version
+ rollback:
+ - createIndex:
+ tableName: sandboxes
+ indexName: idx_sandboxes_dependency_analysis_version
+ columns:
+ - column:
+ name: dependency_analysis_version
+
+ # 18-25. Drop dependency_analysis_version columns from all 8 entity tables
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:17
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from report_card
+ changes:
+ - dropColumn:
+ tableName: report_card
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: report_card
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:18
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from transform
+ changes:
+ - dropColumn:
+ tableName: transform
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: transform
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:19
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from native_query_snippet
+ changes:
+ - dropColumn:
+ tableName: native_query_snippet
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: native_query_snippet
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:20
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from report_dashboard
+ changes:
+ - dropColumn:
+ tableName: report_dashboard
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: report_dashboard
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:21
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from document
+ changes:
+ - dropColumn:
+ tableName: document
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: document
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:22
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from sandboxes
+ changes:
+ - dropColumn:
+ tableName: sandboxes
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: sandboxes
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: smallint
+ defaultValue: 0
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:23
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from segment
+ changes:
+ - dropColumn:
+ tableName: segment
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: segment
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: integer
+ defaultValueNumeric: 0
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v60.2026-04-02T00:00:24
+ author: ericnormand
+ comment: Drop dependency_analysis_version column from measure
+ changes:
+ - dropColumn:
+ tableName: measure
+ columnName: dependency_analysis_version
+ rollback:
+ - addColumn:
+ tableName: measure
+ columns:
+ - column:
+ name: dependency_analysis_version
+ type: integer
+ defaultValueNumeric: 0
+ constraints:
+ nullable: false
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index 07e6933c7a1..3d385d706ec 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -402,8 +402,7 @@
:skip [;; those stats are inherently local state
:view_count :last_viewed_at
;; this is deprecated
- :cache_ttl
- :dependency_analysis_version]
+ :cache_ttl]
:transform {:created_at (serdes/date)
:initially_published_at (serdes/date)
:collection_id (serdes/fk :model/Collection)
diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.clj
index fe10f85de7e..3d4b5a0d42f 100644
--- a/src/metabase/documents/models/document.clj
+++ b/src/metabase/documents/models/document.clj
@@ -192,7 +192,7 @@
(defmethod serdes/make-spec "Document"
[_model-name _opts]
{:copy [:archived :archived_directly :content_type :entity_id :name :collection_position]
- :skip [:view_count :last_viewed_at :public_uuid :made_public_by_id :dependency_analysis_version]
+ :skip [:view_count :last_viewed_at :public_uuid :made_public_by_id]
:transform {:created_at (serdes/date)
:updated_at (serdes/date)
:document {:export-with-context export-document-content
diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.clj
index d457c2ffffd..83e0b59d1f3 100644
--- a/src/metabase/measures/models/measure.clj
+++ b/src/metabase/measures/models/measure.clj
@@ -208,8 +208,7 @@
(defmethod serdes/make-spec "Measure" [_model-name _opts]
{:copy [:name :archived :description :entity_id]
- :skip [:dependency_analysis_version
- ;; dimensions are computed from the query and reconciled on read, not serialized
+ :skip [;; dimensions are computed from the query and reconciled on read, not serialized
:dimensions :dimension_mappings]
:transform {:created_at (serdes/date)
:table_id (serdes/fk :model/Table)
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index 1dac8950d36..68209f8ae52 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -42,6 +42,7 @@
:model/Database metabase.warehouses.models.database
:model/DatabaseRouter metabase-enterprise.database-routing.models
:model/Dependency metabase-enterprise.dependencies.models.dependency
+ :model/DependencyStatus metabase-enterprise.dependencies.models.dependency-status
:model/Dimension metabase.warehouse-schema.models.dimension
:model/Document metabase.documents.models.document
:model/DocumentBookmark metabase.bookmarks.models.bookmark
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index 72e8d132f70..22cf75d276c 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -173,7 +173,7 @@
(defmethod serdes/make-spec "NativeQuerySnippet" [_model-name _opts]
{:copy [:archived :content :description :entity_id :name :template_tags]
- :skip [:dependency_analysis_version]
+ :skip []
:transform {:created_at (serdes/date)
:collection_id (serdes/fk :model/Collection)
:creator_id (serdes/fk :model/User)}
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index e1f1b9dc9f3..520ab07ceb3 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -1320,8 +1320,6 @@
:dataset_query_metrics_v2_migration_backup
;; this column is not used anymore
:cache_ttl
- ;; dependencies aren't serialized, so the version of dependency analysis done shouldn't be serialized
- :dependency_analysis_version
;; dimensions are computed from the query and reconciled on read, not serialized
:dimensions :dimension_mappings
;; temporary column to power rollback from v57 to v56; we can remove it in v58
diff --git a/src/metabase/revisions/impl/dashboard.clj b/src/metabase/revisions/impl/dashboard.clj
index c71d00a9cce..b66cb754039 100644
--- a/src/metabase/revisions/impl/dashboard.clj
+++ b/src/metabase/revisions/impl/dashboard.clj
@@ -18,7 +18,6 @@
;; > The position this Dashboard should appear in the Dashboards list,
;; lower-numbered positions appearing before higher numbered ones.
;; TODO: querying on stats we don't have any dashboard that has a position, maybe we could just drop it?
- :dependency_analysis_version
:public_uuid :made_public_by_id
:position :initially_published_at :view_count
:last_viewed_at})
diff --git a/src/metabase/revisions/impl/measure.clj b/src/metabase/revisions/impl/measure.clj
index f2523bf42c3..dab753ecf92 100644
--- a/src/metabase/revisions/impl/measure.clj
+++ b/src/metabase/revisions/impl/measure.clj
@@ -5,7 +5,7 @@
[metabase.revisions.models.revision :as revision]))
(def ^:private excluded-columns-for-measure-revision
- #{:created_at :updated_at :dependency_analysis_version})
+ #{:created_at :updated_at})
(defmethod revision/serialize-instance :model/Measure
[_model _id instance]
diff --git a/src/metabase/revisions/impl/segment.clj b/src/metabase/revisions/impl/segment.clj
index f1dbde9f4c8..e734b2ba257 100644
--- a/src/metabase/revisions/impl/segment.clj
+++ b/src/metabase/revisions/impl/segment.clj
@@ -5,7 +5,7 @@
[metabase.revisions.models.revision :as revision]))
(def ^:private excluded-columns-for-segment-revision
- #{:created_at :updated_at :dependency_analysis_version})
+ #{:created_at :updated_at})
(defmethod revision/serialize-instance :model/Segment
[_model _id instance]
diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.clj
index 8a201b3fe63..6c6eb59ec25 100644
--- a/src/metabase/segments/models/segment.clj
+++ b/src/metabase/segments/models/segment.clj
@@ -227,7 +227,7 @@
(defmethod serdes/make-spec "Segment" [_model-name _opts]
{:copy [:name :points_of_interest :archived :caveats :description :entity_id :show_in_getting_started]
- :skip [:dependency_analysis_version]
+ :skip []
:transform {:created_at (serdes/date)
:table_id (serdes/fk :model/Table)
:creator_id (serdes/fk :model/User)
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 88fcfa8a847..3a747d21218 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -359,7 +359,7 @@
(defmethod serdes/make-spec "Transform"
[_model-name opts]
{:copy [:name :description :entity_id :owner_email]
- :skip [:dependency_analysis_version :source_type :target_db_id :target_table_id :last_checkpoint_value]
+ :skip [:source_type :target_db_id :target_table_id :last_checkpoint_value]
:transform {:created_at (serdes/date)
:creator_id (serdes/fk :model/User)
:owner_user_id (serdes/fk :model/User)
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 9931dac6558..94bdc358fc3 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -86,7 +86,6 @@
[:collection_id [:maybe pos-int?]]
[:target_db_id {:optional true} [:maybe pos-int?]]
[:run_trigger {:optional true} [:maybe :keyword]]
- [:dependency_analysis_version :int]
[:creator CreatorResponse]
[:last_run {:optional true} [:maybe TransformLastRunResponse]]
[:tag_ids {:optional true} [:sequential pos-int?]]
diff --git a/src/metabase/transforms_rest/api/transform_job.clj b/src/metabase/transforms_rest/api/transform_job.clj
index 92ca02ad8e4..8329e05de58 100644
--- a/src/metabase/transforms_rest/api/transform_job.clj
+++ b/src/metabase/transforms_rest/api/transform_job.clj
@@ -79,7 +79,6 @@
[:creator_id pos-int?]
[:collection_id [:maybe pos-int?]]
[:run_trigger {:optional true} [:maybe :keyword]]
- [:dependency_analysis_version :int]
[:creator CreatorResponse]])
(api.macros/defendpoint :post "/" :- TransformJobResponse
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index ac9a9b43cd2..1e3731916d4 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -36,6 +36,7 @@
:model/DashboardFavorite
:model/DatabaseRouter
:model/Dependency
+ :model/DependencyStatus
:model/MetabotGroupLimit
:model/MetabotInstanceLimit
:model/MetabotPermissions
diff --git a/test/metabase/documents/models/document_test.clj b/test/metabase/documents/models/document_test.clj
index a081c27ebd5..47860692519 100644
--- a/test/metabase/documents/models/document_test.clj
+++ b/test/metabase/documents/models/document_test.clj
@@ -458,7 +458,7 @@
(let [spec (serdes/make-spec "Document" {})]
(is (= [:archived :archived_directly :content_type :entity_id :name :collection_position]
(:copy spec)))
- (is (= [:view_count :last_viewed_at :public_uuid :made_public_by_id :dependency_analysis_version] (:skip spec)))
+ (is (= [:view_count :last_viewed_at :public_uuid :made_public_by_id] (:skip spec)))
(is (contains? (:transform spec) :created_at))
(is (contains? (:transform spec) :document))
(is (contains? (:transform spec) :updated_at))
diff --git a/test/metabase/revisions/impl/card_test.clj b/test/metabase/revisions/impl/card_test.clj
index c69042ffd99..5db31dab234 100644
--- a/test/metabase/revisions/impl/card_test.clj
+++ b/test/metabase/revisions/impl/card_test.clj
@@ -139,9 +139,6 @@
;; we don't expect a description for this column because it should never change
;; once created by the migration
:dataset_query_metrics_v2_migration_backup
- ;; `dependency_analysis_version` is an internal bookkeeping field. It doesn't affect the
- ;; actual card itself, so no description is necessary.
- :dependency_analysis_version
;; this column is immutable in practice, and doesn't warrant a description
:workspace_id}
col)
diff --git a/test/metabase/revisions/impl/segment_test.clj b/test/metabase/revisions/impl/segment_test.clj
index 4787c226c09..04a07394fb6 100644
--- a/test/metabase/revisions/impl/segment_test.clj
+++ b/test/metabase/revisions/impl/segment_test.clj
@@ -28,7 +28,7 @@
(update :id boolean)
(update :table_id boolean)))))
(testing "excluded columns are not present"
- (is (not-any? #{:created_at :updated_at :dependency_analysis_version}
+ (is (not-any? #{:created_at :updated_at}
(keys (revision/serialize-instance :model/Segment (:id segment) segment)))))))
(deftest ^:parallel diff-segments-test
diff --git a/test/metabase/segments/api_test.clj b/test/metabase/segments/api_test.clj
index 2232af82cf5..f8de61f85ce 100644
--- a/test/metabase/segments/api_test.clj
+++ b/test/metabase/segments/api_test.clj
@@ -21,7 +21,7 @@
(defn- segment-response [segment]
(-> (into {} segment)
- (dissoc :id :table_id :dependency_analysis_version)
+ (dissoc :id :table_id)
(update :creator #(into {} %))
(update :entity_id some?)
(update :created_at some?)