Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Recalculate dependencies of entities loaded by serdes and remote sync (#84313)

Alexander Polyankinda6911ee1668
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 349d9e85848..f9c3a3d2a90 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -890,6 +890,11 @@
:api #{metabase.database-routing.core}
:uses #{premium-features}}
+ dependencies
+ {:team "Graphy"
+ :api #{metabase.dependencies.hooks}
+ :uses #{premium-features}}
+
documents
{:team "UX West"
:api #{metabase.documents.api
@@ -1749,6 +1754,7 @@
metabase.models.visualization-settings}
:uses #{api
app-db
+ dependencies
lib
lib.legacy-mbql
lib.metadata
@@ -2014,7 +2020,7 @@
:uses #{analytics.interface
analytics.internal-stats
app-db
- events
+ dependencies
settings
startup
task
@@ -2827,6 +2833,7 @@
app-db
audit-app
database-routing
+ dependencies
driver
events
interestingness
@@ -3058,6 +3065,7 @@
:model/TransformRun}
:uses #{app-db
database-routing
+ dependencies
driver
events
lib
@@ -3886,11 +3894,11 @@
:api #{metabase-enterprise.dependencies.api metabase-enterprise.dependencies.init metabase-enterprise.dependencies.models.dependency}
:uses #{api
app-db
+ batch-processing
collections
database-routing
documents
driver
- events
lib
lib.be
lib.metadata
@@ -3918,13 +3926,20 @@
:model-imports #{:model/Card
:model/Collection
:model/Dashboard
+ :model/DashboardCard
+ :model/DashboardCardSeries
+ :model/DashboardTab
+ :model/Database
+ :model/Dimension
:model/Document
+ :model/Field
:model/FieldUserSettings
:model/Measure
:model/NativeQuerySnippet
:model/Sandbox
:model/Segment
:model/Table
+ :model/TableUserSettings
:model/Transform}}
enterprise/dependencies.replacement
@@ -4210,7 +4225,6 @@
app-db
audit-app
driver
- events
lib
lib.be
lib.legacy-mbql
@@ -4409,7 +4423,7 @@
app-db
appearance
collections
- events
+ dependencies
lib
lib.schema
logger
diff --git a/e2e/test/scenarios/admin/datamodel/segments.cy.spec.ts b/e2e/test/scenarios/admin/datamodel/segments.cy.spec.ts
index 37c004b62f6..5f0336e190a 100644
--- a/e2e/test/scenarios/admin/datamodel/segments.cy.spec.ts
+++ b/e2e/test/scenarios/admin/datamodel/segments.cy.spec.ts
@@ -391,7 +391,7 @@ describe("scenarios > admin > datamodel > segments", () => {
H.activateToken("pro-self-hosted");
cy.log("set up remote sync");
H.setupGitSync();
- H.configureGit("read-only");
+ H.configureGitAndPullChanges("read-only");
H.createLibrary();
H.publishTables({ table_ids: [ORDERS_ID] });
// Let's leave PEOPLE_ID unpublished
diff --git a/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.ts b/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.ts
index 7bb298961c3..39b70689910 100644
--- a/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.ts
+++ b/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.ts
@@ -425,7 +425,7 @@ describe("scenarios > data studio > library", () => {
describe("read-only mode", () => {
beforeEach(() => {
H.setupGitSync();
- H.configureGit("read-only");
+ H.configureGitAndPullChanges("read-only");
H.createLibrary();
});
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/async.clj b/enterprise/backend/src/metabase_enterprise/dependencies/async.clj
deleted file mode 100644
index 8f24b0c3588..00000000000
--- a/enterprise/backend/src/metabase_enterprise/dependencies/async.clj
+++ /dev/null
@@ -1,24 +0,0 @@
-(ns metabase-enterprise.dependencies.async
- "Single-threaded executor for dependency event work.
-
- The two expensive post-save handlers (`::check-card-dependents` and `::update-card-dependents-metadata`)
- submit their work here so the HTTP response returns immediately. Tasks execute serially
- to avoid concurrent writes to dependent cards."
- (:import
- (java.util.concurrent Callable Executors ExecutorService ThreadFactory)))
-
-(set! *warn-on-reflection* true)
-
-(defonce ^:private executor
- (delay (Executors/newSingleThreadExecutor
- (reify ThreadFactory
- (newThread [_ r]
- (doto (Thread. r)
- (.setName "dependency-event-worker")
- (.setDaemon true)))))))
-
-(defn submit!
- "Submit dependency work to the single-threaded executor. Tasks execute serially."
- [f]
- (let [task (bound-fn* f)]
- (.submit ^ExecutorService @executor ^Callable task)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index f84a1df56b4..620d2c59c6e 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -12,7 +12,6 @@
[metabase.lib.schema.id :as lib.schema.id]
[metabase.models.interface :as mi]
[metabase.permissions.core :as perms]
- [metabase.queries.schema :as queries.schema]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
@@ -590,12 +589,6 @@
[card-ids :- [:sequential ::lib.schema.id/card]]
(t2/select [:model/Card :id :database_id] :id [:in card-ids]))
-(mu/defn set-card-result-metadata!
- "Set the result metadata of the Card with `card-id`, returning the number updated."
- [card-id :- ::lib.schema.id/card
- result-metadata :- [:maybe ::queries.schema/card.result-metadata]]
- (t2/update! :model/Card card-id {:result_metadata result-metadata}))
-
(mu/defn tables
"The Tables with `table-ids`."
[table-ids :- [:set ::lib.schema.id/table]]
@@ -606,12 +599,15 @@
[table-ids :- [:sequential ::lib.schema.id/table]]
(t2/select [:model/Table :id :db_id] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
-(mu/defn table-id-by-name
- "The ID of the Table named `table-name` in `schema` of the Database with `db-id`, or nil."
- [db-id :- ::lib.schema.id/database
- schema :- [:maybe :string]
- table-name :- :string]
- (t2/select-one-fn :id :model/Table :db_id db-id :schema schema :name table-name {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))
+(mu/defn dashcard-dashboard-id :- [:maybe ::lib.schema.id/dashboard]
+ "The `:dashboard_id` of the DashboardCard with `dashcard-id`, or nil."
+ [dashcard-id :- ::lib.schema.id/dashcard]
+ (t2/select-one-fn :dashboard_id :model/DashboardCard :id dashcard-id))
+
+(mu/defn field-table-id :- [:maybe ::lib.schema.id/table]
+ "The `:table_id` of the Field with `field-id`, or nil."
+ [field-id :- ::lib.schema.id/field]
+ (t2/select-one-fn :table_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))
(mu/defn transform-sources
"The `:id` and `:source` of the Transforms with `transform-ids`."
@@ -684,10 +680,10 @@
:to_entity_type to-type :to_entity_id to-id))
(mu/defn delete-dependencies-from!
- "Delete the Dependencies of the entity `entity-type` `entity-id`, returning the number deleted."
+ "Delete the Dependencies of the entities `entity-type` `entity-ids`, returning the number deleted."
[entity-type :- EntityType
- entity-id :- ms/PositiveInt]
- (t2/delete! :model/Dependency :from_entity_type entity-type :from_entity_id entity-id))
+ entity-ids :- [:sequential ms/PositiveInt]]
+ (t2/delete! :model/Dependency :from_entity_type entity-type :from_entity_id [:in entity-ids]))
(mu/defn downstream-table-ids-of-transform
"The IDs of the Tables that depend on the Transform with `transform-id`."
@@ -716,11 +712,25 @@
entity-id :- ms/PositiveInt]
(t2/select-one :model/DependencyStatus :entity_type entity-type :entity_id entity-id))
-(mu/defn delete-dependency-status!
- "Delete the DependencyStatus of the entity `entity-type` `entity-id`, returning the number deleted."
+(mu/defn delete-dependency-statuses!
+ "Delete the DependencyStatuses of the entities `entity-type` `entity-ids`, returning the number deleted."
[entity-type :- EntityType
- entity-id :- ms/PositiveInt]
- (t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id entity-id))
+ entity-ids :- [:sequential ms/PositiveInt]]
+ (t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id [:in entity-ids]))
+
+(mu/defn deleted-entity-ids :- [:sequential ms/PositiveInt]
+ "Up to `batch-size` IDs of deleted entities of the type `entity-type` that still have a DependencyStatus."
+ [entity-type :- ::deps.dependency-types/dependency-types
+ batch-size :- ms/PositiveInt]
+ (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))]
+ (vec (t2/select-fn-vec :entity_id :model/DependencyStatus
+ {:where [:and
+ [:= :dependency_status.entity_type (name entity-type)]
+ [:not [:exists ^:allow-subquery
+ {:select [1]
+ :from [[table-name :entity]]
+ :where [:= :entity.id :dependency_status.entity_id]}]]]
+ :limit batch-size}))))
(mu/defn mark-dependency-status-stale!
"Mark the DependencyStatus of `entity-type` `entity-id` as stale for dependency recalculation, creating it if it
@@ -793,7 +803,9 @@
[:or
[:is :dependency_status.next_retry_at nil]
[:<= :dependency_status.next_retry_at now]]]]
- :order-by [[[:case [:= :dependency_status.stale true] [:inline 0] :else [:inline 1]]]]
+ :order-by [[[:case
+ [:or [:= :dependency_status.entity_id nil] [:= :dependency_status.stale true]] [:inline 0]
+ :else [:inline 1]]]]
:limit batch-size})))
;;; ------------------------------------------------ Analysis findings ------------------------------------------------
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
deleted file mode 100644
index 4e10be69122..00000000000
--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj
+++ /dev/null
@@ -1,332 +0,0 @@
-(ns metabase-enterprise.dependencies.events
- (:require
- [metabase-enterprise.dependencies.db :as dependencies.db]
- [metabase-enterprise.dependencies.findings :as deps.findings]
- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
- [metabase-enterprise.dependencies.models.dependency :as models.dependency]
- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
- [metabase-enterprise.dependencies.task.backfill :as task.backfill]
- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
- [metabase.events.core :as events]
- [metabase.premium-features.core :as premium-features :refer [defenterprise]]
- [metabase.util.log :as log]
- [methodical.core :as methodical]))
-
-;; ## Maintaining the dependency graph
-;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the
-;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created
-;; or changed at that point.
-;;
-;; Create/update handlers mark entities stale in dependency_status. The backfill task does the actual computation.
-
-(defn- mark-stale-and-trigger!
- "Mark an entity as stale in dependency_status and trigger the backfill job."
- [entity-type entity-id]
- (try
- (deps.dependency-status/mark-stale! entity-type [entity-id])
- (task.backfill/trigger-backfill-job!)
- (catch Throwable e
- (log/error "Failed to mark entity stale" {:entity-type entity-type :entity-id entity-id :error (ex-message e)}))))
-
-;; ### Cards
-(events/derive! ::card-deps :metabase/event)
-(events/derive! :event/card-create ::card-deps)
-(events/derive! :event/card-update ::card-deps)
-
-(methodical/defmethod events/publish-event! ::card-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :card (:id object))))
-
-(events/derive! ::card-delete :metabase/event)
-(events/derive! :event/card-delete ::card-delete)
-
-(methodical/defmethod events/publish-event! ::card-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :card (:id object))
- (dependencies.db/delete-dependency-status! :card (:id object))))
-
-;; ### Snippets
-(events/derive! ::snippet-deps :metabase/event)
-(events/derive! :event/snippet-create ::snippet-deps)
-(events/derive! :event/snippet-update ::snippet-deps)
-
-(methodical/defmethod events/publish-event! ::snippet-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :snippet (:id object))))
-
-(events/derive! ::snippet-delete :metabase/event)
-(events/derive! :event/snippet-delete ::snippet-delete)
-
-(methodical/defmethod events/publish-event! ::snippet-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :snippet (:id object))
- (dependencies.db/delete-dependency-status! :snippet (:id object))))
-
-;; ### Transforms
-(events/derive! ::transform-deps :metabase/event)
-(events/derive! :event/create-transform ::transform-deps)
-(events/derive! :event/update-transform ::transform-deps)
-
-(methodical/defmethod events/publish-event! ::transform-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :transform (:id object))))
-
-(events/derive! ::transform-delete :metabase/event)
-(events/derive! :event/delete-transform ::transform-delete)
-
-(methodical/defmethod events/publish-event! ::transform-delete
- [_ {:keys [id]}]
- ;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :transform id)
- (dependencies.db/delete-dependency-status! :transform id)))
-
-;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.
-;; (And if the target has changed, the old table's dep on the transform is dropped.)
-;; The upstream deps of the transform are not touched - those change only when the transform is edited.
-(events/derive! ::transform-run :metabase/event)
-(events/derive! :event/transform-run-complete ::transform-run)
-
-(defn- transform-table-deps! [{:keys [db-id output-schema output-table transform-id] :as _details}]
- (let [;; output-table is a keyword like :my_schema/my_table
- table-name (name output-table)]
- (when-let [table-id (dependencies.db/table-id-by-name db-id output-schema table-name)]
- (models.dependency/replace-dependencies! :table table-id {:transform #{transform-id}}))))
-
-(methodical/defmethod events/publish-event! ::transform-run
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (transform-table-deps! object)))
-
-;; ### Dashboards
-(events/derive! ::dashboard-deps :metabase/event)
-(events/derive! :event/dashboard-create ::dashboard-deps)
-(events/derive! :event/dashboard-update ::dashboard-deps)
-
-(methodical/defmethod events/publish-event! ::dashboard-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :dashboard (:id object))))
-
-(events/derive! ::dashboard-delete :metabase/event)
-(events/derive! :event/dashboard-delete ::dashboard-delete)
-
-(methodical/defmethod events/publish-event! ::dashboard-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :dashboard (:id object))
- (dependencies.db/delete-dependency-status! :dashboard (:id object))))
-
-;; ### Documents
-(events/derive! ::document-deps :metabase/event)
-(events/derive! :event/document-create ::document-deps)
-(events/derive! :event/document-update ::document-deps)
-
-(methodical/defmethod events/publish-event! ::document-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :document (:id object))))
-
-(events/derive! ::document-delete :metabase/event)
-(events/derive! :event/document-delete ::document-delete)
-
-(methodical/defmethod events/publish-event! ::document-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :document (:id object))
- (dependencies.db/delete-dependency-status! :document (:id object))))
-
-;; ### Sandboxes
-(events/derive! ::sandbox-deps :metabase/event)
-(events/derive! :event/sandbox-create ::sandbox-deps)
-(events/derive! :event/sandbox-update ::sandbox-deps)
-
-(methodical/defmethod events/publish-event! ::sandbox-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :sandbox (:id object))))
-
-(events/derive! ::sandbox-delete :metabase/event)
-(events/derive! :event/sandbox-delete ::sandbox-delete)
-
-(methodical/defmethod events/publish-event! ::sandbox-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :sandbox (:id object))
- (dependencies.db/delete-dependency-status! :sandbox (:id object))))
-
-;; ### Segments
-(events/derive! ::segment-deps :metabase/event)
-(events/derive! :event/segment-create ::segment-deps)
-(events/derive! :event/segment-update ::segment-deps)
-
-(methodical/defmethod events/publish-event! ::segment-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :segment (:id object))))
-
-(events/derive! ::segment-delete :metabase/event)
-(events/derive! :event/segment-delete ::segment-delete)
-
-(methodical/defmethod events/publish-event! ::segment-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :segment (:id object))
- (dependencies.db/delete-dependency-status! :segment (:id object))))
-
-;; ### Measures
-(events/derive! ::measure-deps :metabase/event)
-(events/derive! :event/measure-create ::measure-deps)
-(events/derive! :event/measure-update ::measure-deps)
-
-(methodical/defmethod events/publish-event! ::measure-deps
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (mark-stale-and-trigger! :measure (:id object))))
-
-(events/derive! ::measure-delete :metabase/event)
-(events/derive! :event/measure-delete ::measure-delete)
-
-(methodical/defmethod events/publish-event! ::measure-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (dependencies.db/delete-dependencies-from! :measure (:id object))
- (dependencies.db/delete-dependency-status! :measure (:id object))))
-
-;; ## Checking dependents for breakage (analysis_finding staleness)
-;;
-;; This is a SEPARATE staleness system from the dependency_status table above.
-;; - dependency_status.stale: tracks whether an entity's upstream *dependency graph* needs recomputation.
-;; Handled by the backfill job (task.backfill).
-;; - analysis_finding staleness: tracks whether an entity's dependents need re-analysis for *breakage detection*
-;; (e.g., a model column was removed — are downstream cards broken?). Handled by the entity-check job
-;; (task.entity-check).
-;;
-;; Both are triggered from the same entity events but serve different purposes and run independently.
-
-(events/derive! ::check-card-dependents :metabase/event)
-(events/derive! :event/card-create ::check-card-dependents)
-(events/derive! :event/card-update ::check-card-dependents)
-
-(methodical/defmethod events/publish-event! ::check-card-dependents
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (deps.findings/mark-entity-and-transitive-dependents-stale! :card (:id object))
- (task.entity-check/trigger-entity-check-job!)))
-
-(events/derive! ::check-card-dependents-on-delete :metabase/event)
-(events/derive! :event/card-delete ::check-card-dependents-on-delete)
-
-(methodical/defmethod events/publish-event! ::check-card-dependents-on-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (when (deps.findings/mark-transitive-dependents-stale! {:card [(:id object)]})
- (task.entity-check/trigger-entity-check-job!))))
-
-(events/derive! ::check-transform :metabase/event)
-(events/derive! :event/create-transform ::check-transform)
-(events/derive! :event/update-transform ::check-transform)
-
-(methodical/defmethod events/publish-event! ::check-transform
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (deps.findings/mark-entity-and-transitive-dependents-stale! :transform (:id object))
- (task.entity-check/trigger-entity-check-job!)))
-
-(events/derive! ::check-transform-on-delete :metabase/event)
-(events/derive! :event/delete-transform ::check-transform-on-delete)
-
-(methodical/defmethod events/publish-event! ::check-transform-on-delete
- [_ {:keys [id]}]
- (when (premium-features/has-feature? :dependencies)
- (when (deps.findings/mark-transitive-dependents-stale! {:transform [id]})
- (task.entity-check/trigger-entity-check-job!))))
-
-(events/derive! ::check-segment-dependents :metabase/event)
-(events/derive! :event/segment-create ::check-segment-dependents)
-(events/derive! :event/segment-update ::check-segment-dependents)
-
-(methodical/defmethod events/publish-event! ::check-segment-dependents
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (deps.findings/mark-entity-and-transitive-dependents-stale! :segment (:id object))
- (task.entity-check/trigger-entity-check-job!)))
-
-(events/derive! ::check-segment-dependents-on-delete :metabase/event)
-(events/derive! :event/segment-delete ::check-segment-dependents-on-delete)
-
-(methodical/defmethod events/publish-event! ::check-segment-dependents-on-delete
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (when (deps.findings/mark-transitive-dependents-stale! {:segment [(:id object)]})
- (task.entity-check/trigger-entity-check-job!))))
-
-(events/derive! ::check-transform-dependents :metabase/event)
-(events/derive! :event/transform-run-complete ::check-transform-dependents)
-
-(methodical/defmethod events/publish-event! ::check-transform-dependents
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (when (deps.findings/mark-transitive-dependents-stale! {:transform [(:transform-id object)]})
- (task.entity-check/trigger-entity-check-job!))))
-
-(defn- synced-db->direct-dependents-of-changed-tables
- "Given the `:db_id` of a freshly synced database, this examines all tables in the DB which were updated, or have
- fields which were updated, since the last time any cards depending on them were analyzed.
-
- It is important that this doesn't re-run the analysis for all dependents of every table whose DB got synced -
- most of the tables have no change every time.
-
- Returns the set of table IDs which have dependents that need re-analysis, possibly empty."
- [db-id]
- (dependencies.db/table-ids-with-outdated-findings db-id))
-
-(events/derive! ::sync-completed-on-database :metabase/event)
-(events/derive! :event/sync-end ::sync-completed-on-database)
-
-(methodical/defmethod events/publish-event! ::sync-completed-on-database
- [_ {db-id :database_id}]
- (when (premium-features/has-feature? :dependencies)
- (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]
- (when (and (seq changes)
- (deps.findings/mark-transitive-dependents-stale! {:table changes}))
- (task.entity-check/trigger-entity-check-job!)))))
-
-;; ### Admin UI Table/Field Metadata Updates
-;; When a table or field's metadata is updated via the admin UI, re-analyze all dependents of that table.
-(events/derive! ::check-table-metadata-update :metabase/event)
-(events/derive! :event/table-update ::check-table-metadata-update)
-
-(methodical/defmethod events/publish-event! ::check-table-metadata-update
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (when (deps.findings/mark-transitive-dependents-stale! {:table [(:id object)]})
- (task.entity-check/trigger-entity-check-job!))))
-
-(events/derive! ::check-field-metadata-update :metabase/event)
-(events/derive! :event/field-update ::check-field-metadata-update)
-
-(methodical/defmethod events/publish-event! ::check-field-metadata-update
- [_ {:keys [object]}]
- (when (premium-features/has-feature? :dependencies)
- (when (deps.findings/mark-transitive-dependents-stale! {:table [(:table_id object)]})
- (task.entity-check/trigger-entity-check-job!))))
-
-;; ### Database Deletion (orphans transforms)
-;; Transforms whose `source_database_id` matches the database being deleted survive the delete
-;; but their existing `analysis_finding` rows still say "OK". Mark them stale here so the entity-check
-;; job re-runs and surfaces them on `/monitor/dependency-diagnostics/broken`.
-(defenterprise mark-transforms-stale-on-database-delete!
- "Enterprise implementation: mark all transforms whose source database is being deleted as stale
- for dependency re-analysis. See the OSS declaration in `metabase.warehouses.models.database`."
- :feature :dependencies
- [db-id]
- (when-let [transform-ids (not-empty (dependencies.db/transform-ids-of-source-database db-id))]
- (deps.analysis-finding/mark-stale! :transform transform-ids)
- (deps.findings/mark-transitive-dependents-stale! {:transform transform-ids})
- (task.entity-check/trigger-entity-check-job!)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
index 4bedc588254..990c09a3d52 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj
@@ -121,8 +121,7 @@
(into {}
(filter (fn [[dep-type _]] (analyzable-entities dep-type)))
pass-through-dependents))]
- (doseq [[dep-type dep-ids] all-supported]
- (deps.analysis-finding/mark-stale! dep-type dep-ids))
+ (deps.analysis-finding/mark-stale! all-supported)
(boolean (seq all-supported))))
(defn mark-transitive-dependents-stale!
@@ -142,7 +141,7 @@
"Mark a single entity as stale for re-analysis by the background job."
[entity-type :- AnalyzableEntityType
entity-id :- pos-int?]
- (deps.analysis-finding/mark-stale! entity-type [entity-id]))
+ (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))
(mu/defn mark-entity-and-transitive-dependents-stale!
"Mark `entity-id` and all of its transitive dependents stale for re-analysis. Use this when an entity changes: it
@@ -162,7 +161,7 @@
This only *analyzes* (each [[upsert-analysis!]] clears that entity's stale flag); it does NOT propagate staleness to
dependents. Propagation happens up front when an entity changes (see [[mark-entity-and-transitive-dependents-stale!]] and the
- event handlers), so the entity-check job drains a fixed set of stale entities that only shrinks — guaranteeing
+ backfill job), so the entity-check job drains a fixed set of stale entities that only shrinks — guaranteeing
termination even when the dependency graph has cycles."
[type :- AnalyzableEntityType
batch-size :- pos-int?]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
new file mode 100644
index 00000000000..69cd1891d49
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj
@@ -0,0 +1,239 @@
+(ns metabase-enterprise.dependencies.hooks
+ "Implements the hooks of `metabase.dependencies.hooks`, marking written content for the background jobs to recompute."
+ (:require
+ [metabase-enterprise.dependencies.db :as dependencies.db]
+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
+ [metabase-enterprise.dependencies.findings :as deps.findings]
+ [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
+ [metabase-enterprise.dependencies.task.backfill :as task.backfill]
+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
+ [metabase.app-db.core :as mdb]
+ [metabase.batch-processing.core :as grouper]
+ [metabase.models.interface :as mi]
+ [metabase.premium-features.core :refer [defenterprise]]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- trigger-jobs-now!
+ "Trigger the backfill and entity-check jobs, logging rather than throwing a failure."
+ []
+ (try
+ (task.backfill/trigger-backfill-job!)
+ (task.entity-check/trigger-entity-check-job!)
+ (catch Throwable e
+ (log/warnf "Failed to trigger the dependency jobs: %s" (ex-message e)))))
+
+(defn- process-writes!
+ "Mark the dependents of the Tables changed by `writes` for re-analysis, then trigger the jobs."
+ [writes]
+ (when-let [table-ids (not-empty (into #{} (keep :table-id) writes))]
+ (deps.findings/mark-transitive-dependents-stale! {:table table-ids}))
+ (trigger-jobs-now!))
+
+(def ^:private writes-queue
+ (delay (grouper/start! process-writes! :capacity 500 :interval 1000)))
+
+(defn- submit-after-commit!
+ "Submit `write` to the batch processed once the current transaction commits."
+ [write]
+ (mdb/do-after-commit #(grouper/submit! @writes-queue write)))
+
+(defn- trigger-jobs!
+ "Have the backfill and entity-check jobs triggered for this write, once it commits, together with the others in its
+ batch, unless a serdes load triggers them once it finishes."
+ []
+ (when-not mi/*deserializing?*
+ (submit-after-commit! :ok)))
+
+(defn- mark-changed!
+ "Mark the entity `entity-type` `entity-id` for recalculation and, when analyzable, for re-analysis; the backfill job
+ marks its dependents for re-analysis."
+ [entity-type entity-id]
+ (when entity-id
+ (deps.dependency-status/mark-stale! {entity-type #{entity-id}})
+ (when (contains? deps.findings/analyzable-entities entity-type)
+ (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))
+ (trigger-jobs!)))
+
+(defn- changed?
+ "Whether the Toucan update of `instance` changes any of `columns`."
+ [instance columns]
+ (let [changes (t2/changes instance)]
+ (some #(contains? changes %) columns)))
+
+(defn- mark-entity-changed!
+ "Mark the `model` entity `instance` for recalculation when its update changes any of `columns`."
+ [model instance columns]
+ (when (changed? instance columns)
+ (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))
+
+(defmulti ^:private before-insert*
+ "Mark what inserting the `model` row `instance` affects."
+ {:arglists '([model instance])}
+ (fn [model _instance] model))
+
+(defmulti ^:private before-update*
+ "Mark what updating the `model` row `instance` affects."
+ {:arglists '([model instance])}
+ (fn [model _instance] model))
+
+(defmulti ^:private before-delete*
+ "Mark what deleting the `model` row `instance` affects."
+ {:arglists '([model instance])}
+ (fn [model _instance] model))
+
+(defmethod before-insert* :default [_model _instance] nil)
+(defmethod before-update* :default [_model _instance] nil)
+(defmethod before-delete* :default [_model _instance] nil)
+
+(doseq [model [:model/Card :model/Dashboard :model/Document :model/Measure :model/NativeQuerySnippet :model/Sandbox
+ :model/Segment :model/Transform]]
+ (defmethod before-insert* model
+ [_model _instance]
+ (trigger-jobs!))
+ (defmethod before-delete* model
+ [model instance]
+ (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))
+
+(doseq [[model columns] {:model/Card #{:type :dataset_query :dimension_mappings :parameters}
+ :model/Dashboard #{:parameters}
+ :model/Document #{:content_type :document}
+ :model/Measure #{:definition :table_id}
+ :model/NativeQuerySnippet #{:content :template_tags}
+ :model/Sandbox #{:card_id}
+ :model/Segment #{:definition :table_id}
+ :model/Transform #{:source :target}}]
+ (defmethod before-update* model
+ [model instance]
+ (mark-entity-changed! model instance columns)))
+
+(defmethod before-insert* :model/DashboardCard
+ [_model dashcard]
+ (mark-changed! :dashboard (:dashboard_id dashcard)))
+
+(defmethod before-update* :model/DashboardCard
+ [_model dashcard]
+ (when (changed? dashcard #{:card_id :dashboard_id :visualization_settings})
+ (doseq [dashboard-id (distinct [(:dashboard_id (t2/original dashcard)) (:dashboard_id dashcard)])]
+ (mark-changed! :dashboard dashboard-id))))
+
+(defmethod before-delete* :model/DashboardCard
+ [_model dashcard]
+ (mark-changed! :dashboard (:dashboard_id dashcard)))
+
+(defmethod before-insert* :model/DashboardCardSeries
+ [_model series]
+ (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))
+
+(defmethod before-update* :model/DashboardCardSeries
+ [_model series]
+ (when (changed? series #{:card_id :dashboardcard_id})
+ (doseq [dashcard-id (distinct [(:dashboardcard_id (t2/original series)) (:dashboardcard_id series)])]
+ (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id dashcard-id)))))
+
+(defmethod before-delete* :model/DashboardCardSeries
+ [_model series]
+ (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))
+
+(defmethod before-delete* :model/DashboardTab
+ [_model tab]
+ (mark-changed! :dashboard (:dashboard_id tab)))
+
+(defn- mark-table-changed!
+ "Mark the dependents of the Table with `table-id` for re-analysis once the current transaction commits."
+ [table-id]
+ (when table-id
+ (submit-after-commit! {:table-id table-id})))
+
+(defmethod before-insert* :model/TableUserSettings
+ [_model settings]
+ (mark-table-changed! (:table_id settings)))
+
+(defmethod before-update* :model/TableUserSettings
+ [_model settings]
+ (mark-table-changed! (:table_id settings)))
+
+(defmethod before-delete* :model/TableUserSettings
+ [_model settings]
+ (mark-table-changed! (:table_id settings)))
+
+(defmethod before-insert* :model/FieldUserSettings
+ [_model settings]
+ (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))
+
+(defmethod before-update* :model/FieldUserSettings
+ [_model settings]
+ (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))
+
+(defmethod before-delete* :model/FieldUserSettings
+ [_model settings]
+ (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))
+
+(defmethod before-insert* :model/Dimension
+ [_model dimension]
+ (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))
+
+(defmethod before-update* :model/Dimension
+ [_model dimension]
+ (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))
+
+(defmethod before-delete* :model/Dimension
+ [_model dimension]
+ (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))
+
+(defmethod before-delete* :model/Database
+ [_model database]
+ (doseq [transform-id (dependencies.db/transform-ids-of-source-database (:id database))]
+ (mark-changed! :transform transform-id)))
+
+(defenterprise before-insert
+ "Mark what inserting the `model` row `instance` affects for recalculation."
+ :feature :dependencies
+ [model instance]
+ (before-insert* model instance))
+
+(defenterprise before-update
+ "Mark what updating the `model` row `instance` affects for recalculation."
+ :feature :dependencies
+ [model instance]
+ (before-update* model instance))
+
+(defenterprise before-delete
+ "Mark what deleting the `model` row `instance` affects for recalculation."
+ :feature :dependencies
+ [model instance]
+ (before-delete* model instance))
+
+(defenterprise after-database-sync
+ "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed."
+ :feature :dependencies
+ [database-id]
+ (when (deps.findings/mark-transitive-dependents-stale!
+ {:table (dependencies.db/table-ids-with-outdated-findings database-id)})
+ (trigger-jobs!)))
+
+(defenterprise after-transform-run
+ "Make the output Table with `table-id` depend on the transform with `transform-id`, and mark the transform's
+ dependents for re-analysis."
+ :feature :dependencies
+ [transform-id table-id]
+ (when table-id
+ (models.dependency/replace-dependencies! :table table-id {:transform #{transform-id}}))
+ (when (deps.findings/mark-transitive-dependents-stale! {:transform #{transform-id}})
+ (trigger-jobs!)))
+
+(defenterprise after-serdes-load
+ "Trigger the backfill and entity-check jobs for what a serdes load marked."
+ :feature :dependencies
+ []
+ (trigger-jobs-now!))
+
+(defenterprise after-premium-token-update
+ "Restart the backfill and entity-check jobs, which stop while the dependencies feature is disabled."
+ :feature :dependencies
+ []
+ (trigger-jobs-now!))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
index 0980db3bfd7..29087660a17 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj
@@ -1,7 +1,6 @@
(ns metabase-enterprise.dependencies.init
(:require
- [metabase-enterprise.dependencies.events]
- [metabase-enterprise.dependencies.metadata-update]
+ [metabase-enterprise.dependencies.hooks]
[metabase-enterprise.dependencies.schema]
[metabase-enterprise.dependencies.settings]
[metabase-enterprise.dependencies.task.backfill]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
index 33343dc604a..55815a561c6 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj
@@ -78,7 +78,7 @@
(declare all-overrides setup-transform! setup-transforms!)
-(deftype OverridingMetadataProvider [delegate *overrides returned-columns-fn]
+(deftype OverridingMetadataProvider [delegate *overrides]
lib.metadata.protocols/MetadataProvider
(database [_this] (lib.metadata.protocols/database delegate))
(metadatas [this metadata-spec]
@@ -121,25 +121,18 @@
(assoc-in m ks v))
% kvs)))
-(defn- returned-columns [^OverridingMetadataProvider mp queryable]
- (let [returned-columns-fn (.returned-columns-fn mp)]
- (or (and returned-columns-fn (returned-columns-fn mp queryable))
- (deps.analysis/returned-columns (:engine (lib.metadata/database mp))
- (lib/query mp queryable)))))
+(defn- returned-columns [mp queryable]
+ (deps.analysis/returned-columns (:engine (lib.metadata/database mp))
+ (lib/query mp queryable)))
(defmethod add-override :card [^OverridingMetadataProvider mp _entity-type id updates]
(with-overrides mp
- ;; If the `updates` contain `:result-metadata`, we want to use that. Similarly, if the user provides a way to
- ;; calculate `result-metadata`, we should use that function. However, any `:result-metadata` from the inner-mp
- ;; should be ignored.
- {[:metadata/card id] (delay (let [temp (merge (when id
- (-> (inner-mp mp)
- (lib.metadata.protocols/card id)))
- updates)
- result-metadata (or (:result-metadata updates)
- (and (.returned-columns-fn mp)
- (returned-columns mp temp)))]
- (assoc temp :result-metadata result-metadata)))
+ ;; Only `:result-metadata` from the `updates` is used; any from the inner-mp is ignored.
+ {[:metadata/card id] (delay (-> (merge (when id
+ (-> (inner-mp mp)
+ (lib.metadata.protocols/card id)))
+ updates)
+ (assoc :result-metadata (:result-metadata updates))))
;; This uses the outer OMP and so the overrides are visible!
[::card-columns id] (delay (returned-columns mp (lib.metadata/card mp id)))}))
@@ -282,10 +275,8 @@
Important note: all overrides must be added first, before this is used as a `MetadataProvider`. The easiest way to
do that is to pass in updated entities and/or dependent ids, which will be immediately added as overrides."
- [{:keys [base-provider updated-entities dependent-ids returned-columns-fn]}]
- (let [^OverridingMetadataProvider omp (->OverridingMetadataProvider base-provider
- (atom {})
- returned-columns-fn)]
+ [{:keys [base-provider updated-entities dependent-ids]}]
+ (let [^OverridingMetadataProvider omp (->OverridingMetadataProvider base-provider (atom {}))]
(doseq [[entity-type updates] updated-entities
updated-entity updates]
(add-override omp entity-type (:id updated-entity) updated-entity))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj
deleted file mode 100644
index 2e41ede9bb8..00000000000
--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj
+++ /dev/null
@@ -1,209 +0,0 @@
-(ns metabase-enterprise.dependencies.metadata-update
- (:require
- [clojure.core.cache :as cache]
- [medley.core :as m]
- [metabase-enterprise.dependencies.async :as async]
- [metabase-enterprise.dependencies.db :as dependencies.db]
- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
- [metabase-enterprise.dependencies.metadata-provider :as deps.metadata-provider]
- [metabase-enterprise.dependencies.models.dependency :as models.dependency]
- [metabase.events.core :as events]
- [metabase.graph.core :as graph]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.lib.schema.common :as lib.schema.common]
- [metabase.lib.schema.id :as lib.schema.id]
- [metabase.lib.schema.metadata :as lib.schema.metadata]
- [metabase.premium-features.core :as premium-features]
- [metabase.queries.core :as queries]
- [metabase.queries.schema :as queries.schema]
- [metabase.util :as u]
- [metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]
- [methodical.core :as methodical]))
-
-(defn- mbql-graph
- "Returns a graph that is limited to sandboxes, segments, measures, and mbql cards.
-
- All other types of nodes are ignored and are neither included in the graph or traversed to find transitive
- dependents."
- [mp]
- (-> (models.dependency/filtered-graph-dependents nil {:types #{:card :sandbox :segment :measure}})
- (graph/filtered-graph (fn [[type id]]
- (not (and (= type :card)
- (->> (lib.metadata/card mp id)
- (models.dependency/is-native-entity? :card))))))
- graph/cached-graph))
-
-(mu/defn- dependent-mbql-cards :- [:sequential ::lib.schema.id/card]
- "Returns a list of all card dependencies in the transitive children of `[start-type start-id]` using `graph`."
- [graph :- ::graph/graph
- start-type :- ::deps.dependency-types/dependency-types
- start-id :- ::deps.dependency-types/entity-id]
- (let [start [start-type start-id]]
- (->> (graph/transitive graph [[start-type start-id]])
- (keep (fn [[node-type node-id :as node]]
- (when (and (= node-type :card)
- (not= node start))
- node-id))))))
-
-(mr/def ::column-metadata-edits
- "Edits `card-metadata-edits` reports: the keys are QP-results-cased (plain keys snake_case, `:lib/...` keys stay
- kebab-case), since values are looked up straight out of `qp-results-cased-col` maps."
- [:map {:closed true}
- [:id {:optional true} [:maybe ::lib.schema.id/field]]
- [:description {:optional true} [:maybe :string]]
- [:display_name {:optional true} [:maybe :string]]
- [:semantic_type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]
- [:fk_target_field_id {:optional true} [:maybe ::lib.schema.id/field]]
- [:settings {:optional true} [:maybe ::lib.schema.common/visualization-settings]]
- [:visibility_type {:optional true} [:maybe ::lib.schema.metadata/column.visibility-type]]
- [:lib/source-display-name {:optional true} [:maybe :string]]])
-(mr/def ::card-metadata-edits [:map-of :string ::column-metadata-edits])
-(mr/def ::card-list-metadata-edits [:map-of ::lib.schema.id/card ::card-metadata-edits])
-
-(mu/defn- column-metadata-edits :- [:maybe ::column-metadata-edits]
- [editable-keys :- [:sequential :keyword]
- original :- [:maybe :metabase.query-processor.middleware.annotate/qp-results-cased-col]
- calculated :- [:maybe :metabase.query-processor.middleware.annotate/qp-results-cased-col]]
- (when (and original calculated)
- (-> (into {}
- (keep (fn [key]
- (let [calculated-value (calculated key)
- original-value (original key)]
- (when (and calculated-value original-value
- (not= calculated-value original-value))
- [key original-value]))))
- editable-keys)
- not-empty)))
-
-(mu/defn- card-metadata-edits :- [:maybe ::card-metadata-edits]
- [mp :- ::lib.schema.metadata/metadata-providerable
- card-id :- ::lib.schema.id/card]
- (let [card (lib.metadata/card mp card-id)
- original-metadata (:result-metadata card)
- ;; Replace the metadata provider here to ensure that it is using the correct one.
- calculated-metadata (->> card :dataset-query (lib/query mp) queries/infer-metadata)
- calculated-by-name (m/index-by :lib/deduplicated-name calculated-metadata)
- editable-keys (->> (lib/model-preserved-keys false)
- (map u/->snake_case_en))]
- (-> (into {}
- (keep (fn [{name :lib/deduplicated-name :as original}]
- (let [calculated (calculated-by-name name)]
- (when-let [edits (column-metadata-edits editable-keys original calculated)]
- [name edits]))))
- original-metadata)
- not-empty)))
-
-(mu/defn- card-list-metadata-edits :- ::card-list-metadata-edits
- "Takes a metadata provider and a list of card ids, and finds the differences between the actual metadata in the db
- and the computed metadata we would expect to see."
- [mp :- ::lib.schema.metadata/metadata-providerable
- card-ids :- [:sequential ::lib.schema.id/card]]
- (into {}
- (keep (fn [card-id]
- (when-let [edit-map (card-metadata-edits mp card-id)]
- [card-id edit-map])))
- card-ids))
-
-(mu/defn- updated-metadata
- "Given a metadata provider, card, and metadata edit map, calculate new metadata with the appropriate metadata edits.
-
- Takes a metadata provider because the queries' existing metadata provider may not be the one we want to use. The
- parent function should be ensuring that the passed-in metadata provider is correct."
- [mp :- ::lib.schema.metadata/metadata-providerable
- card :- ::lib.schema.metadata/card
- metadata-edits :- ::card-list-metadata-edits]
- (let [new-metadata (->> card :dataset-query (lib/query mp) queries/infer-metadata)
- card-edits (metadata-edits (:id card))]
- (if (= (:type card) :model)
- (for [{name :lib/deduplicated-name :as column} new-metadata]
- (merge column (when card-edits (card-edits name))))
- new-metadata)))
-
-(mu/defn- fresh-mp :- ::lib.schema.metadata/metadata-provider
- "Creates a 'fresh' metadata provider for the given database id with no existing cache."
- [db-id :- ::lib.schema.id/database]
- (lib-be/with-existing-metadata-provider-cache (atom (cache/basic-cache-factory {}))
- (lib-be/application-database-metadata-provider db-id)))
-
-(mu/defn- update-dependent-mbql-cards-metadata!
- [original-mp :- ::lib.schema.metadata/metadata-providerable
- start-type :- ::deps.dependency-types/dependency-types
- start-id :- ::deps.dependency-types/entity-id
- previous-object :- ::queries.schema/card
- metadata-type :- :keyword]
- ;; Notes on metadata providers:
- ;;
- ;; The key thing here is that lib.metadata/card caches very aggressively (as of 2026/01, at least). Instead of
- ;; going through the normal cache mechanism, it grabs the card normally, normalizes the query, and then caches the
- ;; result using a special key. The next time it gets called, it will check for that key first and never actually
- ;; hit the standard card fetch flow at all. Meanwhile, override metadata providers delegate all caching to their
- ;; underlying metadata provider. As a result, an override metadata provider and its underlying metadata provider
- ;; effectively use the same card cache. Overriding the card doesn't affect that cache, because lib.metadata/card
- ;; will effectively ask the underlying metadata provider for the card (using that special key) without ever talking
- ;; to the override metadata provider. Both the override metadata provider and the underlying metadata provider will
- ;; always return whichever version of that card was requested and cached first.
- ;;
- ;; The solution here is three different metadata providers. First, we have the original metadata provider (or
- ;; really, metadata-providerable -- a query is fine here). This doesn't have any special overrides and is used to
- ;; fetch the current state of the world before any updates that happen in this function.
- ;;
- ;; Second, we have a "pre-update" metadata provider. This is an override metadata provider based on a "fresh"
- ;; metadata provider with no cached data. This has overrides intended to match the state of the world before
- ;; whatever update triggered this function call.
- ;;
- ;; Third, we have a "updated" metadata provider. This another override metadata provider based on a fresh metadata
- ;; provider, but this one has no initial overrides. Instead, it is set to use our returned-columns logic and asked
- ;; to re-calculate all of the child cards' result metadata. In this case, caching actually works in our favor --
- ;; the override metadata provider does ask its base provider for the original card, and this would cache the old
- ;; version of the card. However, the override metadata provider does this during a call lib.metadata/card. When
- ;; that "parent" call finishes, it stores the updated version of the card in the cache, and that overwrites the old
- ;; version that was previously cached.
- ;;
- ;; If we ever move to a world where model metadata edits are stored separately, we could and should use those
- ;; calculated edits instead of computing them from scratch. However, existing models still wouldn't have edits, so
- ;; we'd need to keep the existing code around for those existing cards. A backfill job could help here, but we'd
- ;; still need to account for cases where we just upgraded an instance and the backfill job hasn't finished yet.
- (let [db-id (-> (lib.metadata/database original-mp)
- :id)
- graph (mbql-graph original-mp)
- cards (dependent-mbql-cards graph start-type start-id)
- previous-metadata (lib-be/instance->metadata previous-object metadata-type)
- pre-update-mp (deps.metadata-provider/override-metadata-provider
- {:base-provider (fresh-mp db-id)
- :updated-entities {start-type [previous-metadata]}})
- edits (card-list-metadata-edits pre-update-mp cards)
- updated-mp (deps.metadata-provider/override-metadata-provider
- {:base-provider (fresh-mp db-id)
- :returned-columns-fn (fn [mp card]
- (updated-metadata mp card edits))
- :dependent-ids {:card cards}})
- start [start-type start-id]
- updates (->> (graph/transitive-children-of graph [start])
- (graph/keep-children
- (fn [[node-type node-id :as node]]
- (when (and (= node-type :card)
- (not= node start))
- (let [new-metadata (-> (lib.metadata/card updated-mp node-id)
- :result-metadata)
- old-metadata (-> (lib.metadata/card original-mp node-id)
- :result-metadata)]
- (if (= new-metadata old-metadata)
- ::graph/stop
- [node-id new-metadata]))))))]
- (doseq [[card-id new-metadata] updates]
- (dependencies.db/set-card-result-metadata! card-id new-metadata))))
-
-(events/derive! ::update-card-dependents-metadata :metabase/event)
-(events/derive! :event/card-update ::update-card-dependents-metadata)
-
-(methodical/defmethod events/publish-event! ::update-card-dependents-metadata
- [_ {{:keys [id dataset_query]} :object :keys [previous-object]}]
- (when (and (premium-features/has-feature? :dependencies)
- (seq dataset_query)
- (not (lib/any-native-stage? dataset_query)))
- (async/submit!
- (fn []
- (update-dependent-mbql-cards-metadata! dataset_query :card id previous-object :metadata/card)))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
index 792cb8f6586..0a3dfbbd278 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
@@ -76,10 +76,11 @@
1000)
(defn mark-stale!
- "Mark entities of `entity-type` with ids in `entity-ids` and existing findings as stale for re-analysis.
- Entities without existing findings are ignored - they'll be picked up by the normal job flow."
- [entity-type entity-ids]
- (doseq [batch (partition-all mark-stale-batch-size entity-ids)]
+ "Mark the existing findings of the entities in `type->ids`, `{entity-type ids}`, stale for re-analysis. Entities
+ without findings are ignored - they'll be picked up by the normal job flow."
+ [type->ids]
+ (doseq [[entity-type ids] type->ids
+ batch (partition-all mark-stale-batch-size ids)]
(dependencies.db/mark-findings-stale! entity-type batch)))
(defn has-stale-entities?
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
index 0513a1660d2..20f69662c0b 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
@@ -16,11 +16,11 @@
{:entity_type mi/transform-keyword})
(defn mark-stale!
- "Mark entities of `entity-type` with ids in `entity-ids` as stale for dependency recalculation.
- Creates entries if they don't exist, or sets stale=true if they do.
- Resets retry state so previously-failed entities get a fresh chance."
- [entity-type entity-ids]
- (doseq [id entity-ids]
+ "Mark the entities in `type->ids`, `{entity-type ids}`, stale for dependency recalculation, creating their statuses
+ if they don't exist and resetting their retry state so previously-failed entities get a fresh chance."
+ [type->ids]
+ (doseq [[entity-type ids] type->ids
+ id ids]
(dependencies.db/mark-dependency-status-stale! entity-type id)))
(defn upsert-status!
@@ -60,12 +60,14 @@
(defn has-stale-or-outdated?
"Returns true if there are any entities needing dependency calculation: no status row yet,
- stale=true, OR version < current (not terminal, retry delay elapsed). Defined in terms of
- [[instances-for-dependency-calculation]] so it stays consistent with what the backfill processes."
+ stale=true, OR version < current (not terminal, retry delay elapsed), or deleted entities whose status the backfill
+ has yet to clean up. Defined in terms of [[instances-for-dependency-calculation]] so it stays consistent with what
+ the backfill processes."
[]
(boolean
(some (fn [entity-type]
- (seq (instances-for-dependency-calculation entity-type 1)))
+ (or (seq (instances-for-dependency-calculation entity-type 1))
+ (seq (dependencies.db/deleted-entity-ids entity-type 1))))
deps.dependency-types/backfillable-dependency-types)))
(defn record-failure!
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
index 9278b76bdae..3e50587b6eb 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
@@ -5,7 +5,8 @@
This is done by querying the dependency_status table for entities that are stale or have an
outdated dependency_analysis_version. The backfill task computes dependencies and updates
- the dependency_status table."
+ the dependency_status table. Entities that a write marked stale, or that were deleted, also have their dependents
+ marked for re-analysis here, outside the write's transaction."
(:require
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.scheduler :as qs]
@@ -14,17 +15,17 @@
[metabase-enterprise.dependencies.calculation :as deps.calculation]
[metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
+ [metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
[metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
[metabase-enterprise.dependencies.settings :as deps.settings]
[metabase-enterprise.dependencies.task-util :as deps.task-util]
- [metabase.events.core :as events]
+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
[metabase.lib-be.core :as lib-be]
[metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
[metabase.transforms.core :as transforms]
[metabase.util.log :as log]
- [methodical.core :as methodical]
[toucan2.core :as t2])
(:import (org.quartz JobExecutionContext)))
@@ -75,16 +76,31 @@
(deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))
(defn- compute-deps-for-entity!
- "Compute and store dependencies for an entity, then update its dependency_status.
- Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."
+ "Mark the dependents of an entity that a write marked stale for re-analysis, then compute and store its
+ dependencies and update its dependency_status. Entities are expected to be pre-hydrated by
+ [[deps.dependency-status/hydrate-for-deps]]."
[entity-type entity]
(log/debug "Computing dependencies for" (name entity-type) (:id entity))
+ (when (:stale (dependencies.db/dependency-status entity-type (:id entity)))
+ (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}}))
(t2/with-transaction [_]
(let [deps (deps.calculation/calculate-deps entity-type entity)]
(models.dependency/replace-dependencies! entity-type (:id entity) deps)
(post-deps-cleanup! entity-type entity)
(deps.dependency-status/upsert-status! entity-type (:id entity)))))
+(defn- remove-deleted-entity-batch!
+ "Mark the dependents of up to `batch-size` deleted entities of `entity-type` for re-analysis and drop their
+ dependencies and status, returning how many were removed."
+ [entity-type batch-size]
+ (let [entity-ids (dependencies.db/deleted-entity-ids entity-type batch-size)]
+ (when (seq entity-ids)
+ (t2/with-transaction [_]
+ (deps.findings/mark-transitive-dependents-stale! {entity-type entity-ids})
+ (dependencies.db/delete-dependencies-from! entity-type entity-ids)
+ (dependencies.db/delete-dependency-statuses! entity-type entity-ids)))
+ (count entity-ids)))
+
(defn- backfill-entity-batch!
[entity-type batch-size]
(lib-be/with-metadata-provider-cache
@@ -121,20 +137,26 @@
instances))))
(defn- backfill-dependencies!
- "Job to backfill dependencies for all entities.
+ "Job to backfill dependencies for all entities, triggering the entity-check job for the findings it marked stale.
Returns true if a full batch has been selected, nil or false otherwise."
[]
(when (premium-features/has-feature? :dependencies)
- (-> (reduce (fn [batch-size entity-type]
- (if (< batch-size 1)
- (reduced 0)
- (let [processed (backfill-entity-batch! entity-type batch-size)]
- (when (pos? processed)
- (log/info "Updated" processed "entities."))
- (- batch-size processed))))
- (deps.settings/dependency-backfill-batch-size)
- entity-types)
- (< 1))))
+ (let [batch-size (deps.settings/dependency-backfill-batch-size)
+ remaining (reduce (fn [remaining entity-type]
+ (if (< remaining 1)
+ (reduced 0)
+ (let [removed (remove-deleted-entity-batch! entity-type remaining)
+ processed (if (< removed remaining)
+ (backfill-entity-batch! entity-type (- remaining removed))
+ 0)]
+ (when (pos? processed)
+ (log/info "Updated" processed "entities."))
+ (- remaining removed processed))))
+ batch-size
+ entity-types)]
+ (when (< remaining batch-size)
+ (task.entity-check/trigger-entity-check-job!))
+ (< remaining 1))))
(defn- has-pending-retries?
"Whether any entity is in retry backoff that this instance can actually act on.
@@ -227,12 +249,3 @@
(task/scheduler)
(deps.task-util/job-initial-delay
(deps.settings/dependency-backfill-variance-minutes))))
-
-(events/derive! ::backfill :metabase/event)
-(events/derive! :event/serdes-load ::backfill)
-(events/derive! :event/set-premium-embedding-token ::backfill)
-
-(methodical/defmethod events/publish-event! ::backfill
- [_ _]
- (when (premium-features/has-feature? :dependencies)
- (trigger-backfill-job!)))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
index 17c5a849ea2..d7b36c65ce3 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
@@ -7,11 +7,9 @@
[metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
[metabase-enterprise.dependencies.settings :as deps.settings]
[metabase-enterprise.dependencies.task-util :as deps.task-util]
- [metabase.events.core :as events]
[metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
- [metabase.util.log :as log]
- [methodical.core :as methodical]))
+ [metabase.util.log :as log]))
(set! *warn-on-reflection* true)
@@ -30,16 +28,7 @@
(< 1)))
(defn- check-entities!
- "Drain the currently-stale entities, analyzing each one.
-
- Staleness is propagated up front, at the moment an entity changes (see the dependencies event handlers, which mark the
- whole affected subtree via [[deps.findings/mark-entity-and-transitive-dependents-stale!]]). The drain therefore works through a
- fixed set that only shrinks — it never re-marks anything stale during draining — so it always terminates, even when the
- dependency graph has cycles.
-
- As a safety net, the loop also stops as soon as a pass fails to reduce the stale count (e.g. an entity whose analysis
- can't be persisted). That keeps a single unprocessable entity from spinning the loop forever; the next scheduled run
- retries it."
+ "Analyze stale entities in batches until none are left or a pass stops reducing their count."
[]
(when (premium-features/has-feature? :dependencies)
(loop [prev-stale nil]
@@ -52,18 +41,7 @@
(declare schedule-run!)
(defn- reschedule-after-run!
- "Schedule the next periodic run of the entity check job, unless the `:dependencies` feature is absent.
-
- Unlike the backfill job this one has no terminal state — it is a periodic checker, so while licensed it always queues
- another run rather than stopping once the stale set is drained. Without the feature there is nothing the next run
- could do, since [[check-entities!]] is gated on it, so rescheduling would just wake the instance forever. The job
- restarts from the token event above, from the content-change handlers, or on the next restart.
-
- Scheduling deliberately fails open on an indeterminate token status: `has-feature?` cannot tell a network failure
- from being unlicensed, and nothing re-fires when the check later succeeds, so reading a blip as `false` would end
- the chain for good. Only a definitive `false` stops it — which is the common unlicensed case, since no token at all
- answers `false`. A token that cannot be validated answers `nil` and keeps the job scheduled: if there is a token, we
- keep trying. Doing the work still fails closed — see [[check-entities!]]."
+ "Schedule the next run of the entity check job unless the `:dependencies` feature is definitively absent."
[scheduler]
(if (false? (premium-features/canonically-has-feature? :dependencies))
(log/debug "Not rescheduling job Dependency Entity Check: the dependencies feature is not enabled")
@@ -117,14 +95,3 @@
[]
(when (pos? (deps.settings/dependency-entity-check-batch-size))
(schedule-run! (task/scheduler) 1)))
-
-;;; Mirrors the backfill job's wiring: enabling the feature has to restart the job, because
-;;; [[reschedule-after-run!]] stops the periodic chain while it is disabled and the content-change handlers that
-;;; would otherwise revive it are themselves gated on the feature.
-(events/derive! ::entity-check :metabase/event)
-(events/derive! :event/set-premium-embedding-token ::entity-check)
-
-(methodical/defmethod events/publish-event! ::entity-check
- [_ _]
- (when (premium-features/has-feature? :dependencies)
- (trigger-entity-check-job!)))
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj
index 7da5ba2d14c..9cee0d59b82 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj
@@ -12,7 +12,6 @@
[metabase-enterprise.sandbox.schema :as sandbox.schema]
[metabase.api.common :as api]
[metabase.audit-app.core :as audit]
- [metabase.events.core :as events]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.schema.id :as lib.schema.id]
@@ -40,7 +39,8 @@
(derive :metabase/model)
;;; only admins can work with sandboxes
(derive ::mi/read-policy.superuser)
- (derive ::mi/write-policy.superuser))
+ (derive ::mi/write-policy.superuser)
+ (derive :hook/dependencies))
(t2/deftransforms :model/Sandbox
{:attribute_remappings sandbox.schema/attribute-remappings-transform})
@@ -280,16 +280,8 @@
(do
(when (some #(contains? sandbox %) [:card_id :attribute_remappings])
(sandbox.db/update-sandbox! id (u/select-keys-when sandbox :present #{:card_id :attribute_remappings})))
- (let [updated-sandbox (sandbox.db/sandbox id)]
- (events/publish-event! :event/sandbox-update
- {:object updated-sandbox
- :user-id api/*current-user-id*})
- updated-sandbox))
- (let [inserted-sandbox (sandbox.db/insert-sandbox! sandbox)]
- (events/publish-event! :event/sandbox-create
- {:object inserted-sandbox
- :user-id api/*current-user-id*})
- inserted-sandbox)))))
+ (sandbox.db/sandbox id))
+ (sandbox.db/insert-sandbox! sandbox)))))
(defn- normalize-sandbox-attribute-remappings
"Normalize `:attribute_remappings` on a Sandbox map, if present. `before-insert`/`before-update` hooks see this
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj
index 4ebd8d9f284..8ba5fbc41d6 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj
@@ -10,7 +10,6 @@
[metabase-enterprise.serialization.v2.storage.files :as v2.storage.files]
[metabase.analytics.core :as analytics]
[metabase.app-db.core :as mdb]
- [metabase.events.core :as events]
[metabase.models.serialization :as serdes]
[metabase.plugins.core :as plugins]
[metabase.premium-features.core :as premium-features]
@@ -53,9 +52,8 @@
;;(when-not (load/compatible? path)
;; (log/warn "Dump was produced using a different version of Metabase. Things may break!"))
(log/infof "Loading serialized Metabase files from %s" path)
- (u/prog1 (serdes/with-cache
- (v2.load/load-metabase! (v2.ingest/ingest-yaml path) opts))
- (events/publish-event! :event/serdes-load {})))
+ (serdes/with-cache
+ (v2.load/load-metabase! (v2.ingest/ingest-yaml path) opts)))
(mu/defn v2-load!
"SerDes v2 load entry point.
@@ -114,8 +112,6 @@
(serdes/with-cache
(-> (v2.extract/extract opts)
(v2.storage/store! (v2.storage.files/file-writer path))))
- ;; we could publish :event/serdes-dump to go with :event/serdes-load above, but
- ;; nothing would listen to it currently
(catch Exception e
(reset! err e)))]
(analytics/track-event! :snowplow/serialization
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
index 0ff5a078b96..fcbaa74296f 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj
@@ -9,6 +9,8 @@
[metabase-enterprise.serialization.v2.models :as serdes.models]
[metabase.app-db.core :as mdb]
[metabase.app-db.transient-error :as transient-error]
+ [metabase.dependencies.hooks :as dependencies.hooks]
+ [metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.search.core :as search]
[metabase.util :as u]
@@ -223,8 +225,9 @@
(with-retries 3 200
(fn []
(try
- (t2/with-transaction [_tx]
- (serdes/load-one! ingested local-or-nil))
+ (binding [mi/*deserializing?* true]
+ (t2/with-transaction [_tx]
+ (serdes/load-one! ingested local-or-nil)))
(catch Throwable e
(serdes/reset-import-cache!)
(throw e)))))
@@ -285,17 +288,20 @@
:files file-names}
(first ingest-errors)))))
(log/infof "Starting deserialization, total %s documents" (count contents))
- (reduce (fn [ctx item]
- (try
- (load-one! ctx item)
- (catch Exception e
- (when-not continue-on-error
- (throw e))
- ;; eschew big and scary stacktrace
- (log/warnf (u/strip-error e "Skipping deserialization error"))
- (update ctx :errors conj e))))
- ctx
- contents))
+ (try
+ (reduce (fn [ctx item]
+ (try
+ (load-one! ctx item)
+ (catch Exception e
+ (when-not continue-on-error
+ (throw e))
+ ;; eschew big and scary stacktrace
+ (log/warnf (u/strip-error e "Skipping deserialization error"))
+ (update ctx :errors conj e))))
+ ctx
+ contents)
+ (finally
+ (dependencies.hooks/after-serdes-load))))
(when reindex?
;; Reindex after all entities are loaded. Individual entity commits may have produced stale
;; search index entries; this ensures the index reflects the final state.
diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj
index eeaaabe64d1..0e742df9456 100644
--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj
@@ -514,7 +514,7 @@
:creator_id "[email protected]"
:created_at string?
:card_schema pos-int?
- :visualization_settings {:column_settings nil}
+ :visualization_settings {}
:parameters []
:parameter_mappings []
:dataset_query {:database (db-name)
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index aeb454922aa..90dd9755518 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -3,9 +3,8 @@
[clojure.test :refer :all]
[medley.core :as m]
[metabase-enterprise.dependencies.api :as deps.api]
- [metabase-enterprise.dependencies.async :as dependencies.async]
- [metabase-enterprise.dependencies.events]
[metabase-enterprise.dependencies.findings :as dependencies.findings]
+ [metabase-enterprise.dependencies.hooks :as deps.hooks]
[metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.collections.models.collection :as collection]
@@ -23,9 +22,6 @@
[metabase.util :as u]
[toucan2.core :as t2]))
-(comment
- metabase-enterprise.dependencies.events/keep-me)
-
(defn- card-with-query
"Create a card map with the given name and dataset query."
[card-name dataset-query]
@@ -134,19 +130,6 @@
[card-id]
(dependencies.findings/upsert-analysis! (t2/select-one :model/Card :id card-id)))
-; dependencies.async/submit! effectively awaits all pending tasks on the executor.
-; Those tasks would be executed regardless; this is just changing the timing to be
-; less problematic for multiple test runs in sequence.
-#_{:clj-kondo/ignore [:metabase/validate-deftest]}
-(use-fixtures :each
- (fn drained-dependency-async-executor-fixture [t]
- (try
- (t)
- (finally
- ;; Drain the single-threaded async executor to ensure all pending dependency
- ;; work completes before model cleanup deletes cards, avoiding lock timeouts.
- @(dependencies.async/submit! (fn [] nil))))))
-
(deftest graph-test
(testing "GET /api/ee/dependencies/graph"
(mt/with-premium-features #{:dependencies}
@@ -466,11 +449,7 @@
:name "inactive_transform_table"}}]
;; Simulate the transform having been run: link it to its (now inactive) target table
(t2/update! :model/Transform transform-id {:target_table_id table-id})
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "PUBLIC"
- :output-table "inactive_transform_table"
- :transform-id transform-id}})
+ (deps.hooks/after-transform-run transform-id table-id)
(deps.test/synchronously-run-backfill!)
(testing "GET /graph/dependents returns the inactive table as a child"
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"
@@ -1198,11 +1177,7 @@
;; Simulate the referenced transform having been run: link it to its target table
;; so the table→transform dep is visible in the dependency graph.
(t2/update! :model/Transform referenced-transform-id {:target_table_id referenced-table-id})
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "PUBLIC"
- :output-table "referenced_transform_table"
- :transform-id referenced-transform-id}})
+ (deps.hooks/after-transform-run referenced-transform-id referenced-table-id)
(deps.test/synchronously-run-backfill!)
(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")]
(is (=? {:data [{:id unreffed-transform-id
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
index b9f5efadd08..57006cb47d4 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj
@@ -1,23 +1,21 @@
(ns metabase-enterprise.dependencies.events-test
(:require
[clojure.test :refer [deftest is testing]]
- [metabase-enterprise.dependencies.events]
[metabase-enterprise.dependencies.findings :as deps.findings]
+ [metabase-enterprise.dependencies.hooks :as deps.hooks]
[metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding]
[metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
[metabase-enterprise.dependencies.task.entity-check :as task.entity-check]
[metabase-enterprise.dependencies.test-util :as deps.test]
- [metabase.api.common :as api]
- [metabase.events.core :as events]
+ [metabase-enterprise.serialization.test-util :as ts]
+ [metabase-enterprise.serialization.v2.load :as serdes.load]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.models.serialization :as serdes]
[metabase.test :as mt]
[toucan2.core :as t2]))
-(comment
- metabase-enterprise.dependencies.events/keep-me)
-
(defn- run-with-dependencies-setup! [thunk]
(mt/with-test-user :rasta
(mt/with-premium-features #{:dependencies}
@@ -32,6 +30,12 @@
(is (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true)
(str "Expected " (name entity-type) " " entity-id " to be marked stale")))
+(defn- assert-pending
+ "Assert that the given entity awaits the backfill job, having no dependency_status row or a stale one."
+ [entity-type entity-id]
+ (is (not (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale false))
+ (str "Expected " (name entity-type) " " entity-id " to be pending")))
+
(deftest dashboard-update-sets-correct-dependencies
(run-with-dependencies-setup!
(fn [mp]
@@ -45,17 +49,16 @@
:model/Card {param-source-card-id :id} {:dataset_query (-> (lib/query mp products)
(lib/aggregate (lib/count))
(lib/breakout category-field))}
- :model/Dashboard {dashboard-id :id :as dashboard} {:parameters [{:id "category-param"
- :type "category"
- :values_source_type "card"
- :values_source_config {:card_id param-source-card-id}}]}
+ :model/Dashboard {dashboard-id :id} {:parameters [{:id "category-param"
+ :type "category"
+ :values_source_type "card"
+ :values_source_config {:card_id param-source-card-id}}]}
:model/DashboardCard {basic-dashcard-id :id} {:dashboard_id dashboard-id
:card_id basic-card-id}
:model/DashboardCardSeries _ {:dashboardcard_id basic-dashcard-id
:card_id series-card-id
:position 0}]
- (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})
- (assert-stale :dashboard dashboard-id)
+ (assert-pending :dashboard dashboard-id)
(deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :dashboard
:from_entity_id dashboard-id
@@ -111,46 +114,44 @@
:visualization_settings {:column_settings {"[\"name\",\"CATEGORY\"]" {:click_behavior {:type "link"
:linkType "dashboard"
:targetId column-click-target-dashboard-id}}}}}]
- (let [updated-dashboard (t2/select-one :model/Dashboard :id dashboard-id)]
- (events/publish-event! :event/dashboard-update {:object updated-dashboard :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (=? #{{:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :card
- :to_entity_id scalar-card-id}
- {:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :card
- :to_entity_id scalar-click-target-card-id}
- {:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :card
- :to_entity_id scalar-card-2-id}
- {:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :dashboard
- :to_entity_id scalar-click-target-dashboard-id}
- {:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :card
- :to_entity_id table-card-id}
- {:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :card
- :to_entity_id column-click-target-card-id}
- {:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :card
- :to_entity_id table-card-2-id}
- {:from_entity_type :dashboard
- :from_entity_id dashboard-id
- :to_entity_type :dashboard
- :to_entity_id column-click-target-dashboard-id}}
- (into #{} (map #(dissoc % :id)
- (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))
- (t2/delete! :model/Dashboard dashboard-id)
- (events/publish-event! :event/dashboard-delete {:object updated-dashboard :user-id api/*current-user-id*})
- (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard))))))))))
+ (deps.test/synchronously-run-backfill!)
+ (is (=? #{{:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :card
+ :to_entity_id scalar-card-id}
+ {:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :card
+ :to_entity_id scalar-click-target-card-id}
+ {:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :card
+ :to_entity_id scalar-card-2-id}
+ {:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :dashboard
+ :to_entity_id scalar-click-target-dashboard-id}
+ {:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :card
+ :to_entity_id table-card-id}
+ {:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :card
+ :to_entity_id column-click-target-card-id}
+ {:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :card
+ :to_entity_id table-card-2-id}
+ {:from_entity_type :dashboard
+ :from_entity_id dashboard-id
+ :to_entity_type :dashboard
+ :to_entity_id column-click-target-dashboard-id}}
+ (into #{} (map #(dissoc % :id)
+ (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))
+ (t2/delete! :model/Dashboard dashboard-id)
+ (deps.test/synchronously-run-backfill!)
+ (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))))))
(deftest document-update-sets-correct-dependencies
(run-with-dependencies-setup!
@@ -169,8 +170,7 @@
"model" "card"}}]}
{:type "cardEmbed"
:attrs {"id" embedded-card-id}}]}}]
- (events/publish-event! :event/document-create {:object document :user-id api/*current-user-id*})
- (assert-stale :document document-id)
+ (assert-pending :document document-id)
(deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :document
:from_entity_id document-id
@@ -191,7 +191,6 @@
:attrs {"entityId" products-id
"model" "table"}}]}]})]
(t2/update! :model/Document document-id updated-doc)
- (events/publish-event! :event/document-update {:object updated-doc :user-id api/*current-user-id*})
(deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :document
:from_entity_id document-id
@@ -204,7 +203,7 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document)))))
(t2/delete! :model/Document document-id)
- (events/publish-event! :event/document-delete {:object document :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document))))))))))
(deftest sandbox-update-sets-correct-dependencies
@@ -217,8 +216,7 @@
:model/Sandbox {sandbox-id :id :as sandbox} {:group_id group-id
:table_id (mt/id :products)
:card_id card1-id}]
- (events/publish-event! :event/sandbox-create {:object sandbox :user-id api/*current-user-id*})
- (assert-stale :sandbox sandbox-id)
+ (assert-pending :sandbox sandbox-id)
(deps.test/synchronously-run-backfill!)
(is (=? #{{:from_entity_type :sandbox
:from_entity_id sandbox-id
@@ -227,31 +225,33 @@
(into #{} (map #(dissoc % :id)
(t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))
(t2/update! :model/Sandbox sandbox-id (assoc sandbox :card_id card2-id))
- (let [updated-sandbox (t2/select-one :model/Sandbox :id sandbox-id)]
- (events/publish-event! :event/sandbox-update {:object updated-sandbox :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (=? #{{:from_entity_type :sandbox
- :from_entity_id sandbox-id
- :to_entity_type :card
- :to_entity_id card2-id}}
- (into #{} (map #(dissoc % :id)
- (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))
- (t2/delete! :model/Sandbox sandbox-id)
- (events/publish-event! :event/sandbox-delete {:object sandbox :user-id api/*current-user-id*})
- (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))))))
+ (deps.test/synchronously-run-backfill!)
+ (is (=? #{{:from_entity_type :sandbox
+ :from_entity_id sandbox-id
+ :to_entity_type :card
+ :to_entity_id card2-id}}
+ (into #{} (map #(dissoc % :id)
+ (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))
+ (t2/delete! :model/Sandbox sandbox-id)
+ (deps.test/synchronously-run-backfill!)
+ (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))
-(deftest card-marks-stale-on-create-update-test
- (testing "Card create/update events mark the entity stale in dependency_status"
+(deftest card-marks-pending-on-create-and-stale-on-update-test
+ (testing "A created card is pending for the backfill job, and changing its query marks it stale in dependency_status"
(run-with-dependencies-setup!
(fn [mp]
- (let [products (lib.metadata/table mp (mt/id :products))]
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]
- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})
- (assert-stale :card card-id)
+ (let [products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
+ (assert-pending :card card-id)
(deps.test/synchronously-run-backfill!)
(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id (mt/id :products)))
- (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false))
+ (testing "a change to a column the dependencies do not come from leaves it up to date"
+ (t2/update! :model/Card card-id {:name "renamed"})
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false)))
+ (t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)})
(assert-stale :card card-id)))))))
(deftest metric-dimensions-update-marks-stale-and-includes-mapped-table-test
@@ -265,18 +265,11 @@
;; uncurated, so the after-insert dimension auto-sync would seed dimensions from the query and
;; overwrite mappings passed to with-temp. A plain update sticks — the after-update hook only
;; re-syncs when `:dataset_query` changes.
- ;; The dimension endpoints announce their writes as card updates; see
- ;; [[metabase.metrics.api/notify-dimensions-changed!]].
- (let [before (t2/select-one :model/Card :id card-id)]
- (t2/update! :model/Card card-id
- {:dimension_mappings [{:type :table
- :dimension-id "550e8400-e29b-41d4-a716-446655440000"
- :table-id (mt/id :categories)
- :target [:field {} (mt/id :categories :name)]}]})
- (events/publish-event! :event/card-update
- {:object (t2/select-one :model/Card :id card-id)
- :previous-object before
- :user-id api/*current-user-id*}))
+ (t2/update! :model/Card card-id
+ {:dimension_mappings [{:type :table
+ :dimension-id "550e8400-e29b-41d4-a716-446655440000"
+ :table-id (mt/id :categories)
+ :target [:field {} (mt/id :categories :name)]}]})
(assert-stale :card card-id)
(deps.test/synchronously-run-backfill!)
(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id
@@ -286,40 +279,41 @@
:to_entity_type :table :to_entity_id (mt/id :venues))
"the query's own table remains a dependency")))))))
-(deftest snippet-marks-stale-on-create-update-test
- (testing "Snippet create/update events mark the entity stale in dependency_status"
+(deftest snippet-marks-pending-on-create-and-stale-on-update-test
+ (testing "A created snippet is pending for the backfill job, and changing its content marks it stale in dependency_status"
(run-with-dependencies-setup!
(fn [_]
- (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id :as snippet} {:name "test snippet"
- :content "SELECT 1"}]
- (events/publish-event! :event/snippet-create {:object snippet :user-id api/*current-user-id*})
- (assert-stale :snippet snippet-id)
- (events/publish-event! :event/snippet-update {:object snippet :user-id api/*current-user-id*})
+ (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "test snippet"
+ :content "SELECT 1"}]
+ (assert-pending :snippet snippet-id)
+ (deps.test/synchronously-run-backfill!)
+ (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id snippet-id :stale false))
+ (t2/update! :model/NativeQuerySnippet snippet-id {:content "SELECT 2"})
(assert-stale :snippet snippet-id))))))
-(deftest mark-stale-failure-does-not-propagate-test
- (testing "When mark-stale! throws, the event handler catches the error and the API call is not affected"
+(deftest mark-stale-failure-fails-the-write-test
+ (testing "When mark-stale! throws, the update fails and the card keeps its query"
(run-with-dependencies-setup!
(fn [mp]
- (let [products (lib.metadata/table mp (mt/id :products))]
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]
+ (let [products (lib.metadata/table mp (mt/id :products))
+ orders (lib.metadata/table mp (mt/id :orders))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]
+ (deps.test/synchronously-run-backfill!)
(mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!
- (fn [_ _] (throw (ex-info "Simulated DB failure" {})))]
- ;; Should not throw — the error is caught and logged
- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*}))
- ;; Entity should NOT be stale (mark-stale! failed)
- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))
- "Entity should not be marked stale when mark-stale! fails")))))))
+ (fn [_] (throw (ex-info "Simulated DB failure" {})))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated DB failure"
+ (t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)}))))
+ (is (=? {:stages [{:source-table (mt/id :products)}]}
+ (t2/select-one-fn :dataset_query :model/Card card-id)))))))))
-(deftest ^:synchronized native-transform-updates-dependencies-test
- (testing "native transform update events trigger dependency calculations"
+(deftest ^:synchronized native-transform-create-sets-dependencies-test
+ (testing "a created native transform is picked up by the dependency calculation"
(run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/native-query mp "select * from orders")
:type :query}]
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (assert-stale :transform transform-id)
+ (mt/with-temp [:model/Transform {transform-id :id} {:source source}]
+ (assert-pending :transform transform-id)
(deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
@@ -327,15 +321,14 @@
:to_entity_id (mt/id :orders)}]
(t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))
-(deftest ^:synchronized mbql-transform-updates-dependencies-test
- (testing "mbql transform update events trigger dependency calculations"
+(deftest ^:synchronized mbql-transform-create-sets-dependencies-test
+ (testing "a created mbql transform is picked up by the dependency calculation"
(run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders)))
:type :query}]
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (assert-stale :transform transform-id)
+ (mt/with-temp [:model/Transform {transform-id :id} {:source source}]
+ (assert-pending :transform transform-id)
(deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
@@ -343,8 +336,8 @@
:to_entity_id (mt/id :orders)}]
(t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))
-(deftest ^:synchronized python-transform-updates-dependencies-test
- (testing "python transform update events trigger dependency calculations"
+(deftest ^:synchronized python-transform-create-sets-dependencies-test
+ (testing "a created python transform is picked up by the dependency calculation"
(run-with-dependencies-setup!
(fn [_]
(let [source {:type :python,
@@ -356,9 +349,8 @@
:table_id (mt/id :orders)}],
:body
"import pandas as pd\n\ndef transform(orders):\n return orders"}]
- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]
- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
- (assert-stale :transform transform-id)
+ (mt/with-temp [:model/Transform {transform-id :id} {:source source}]
+ (assert-pending :transform transform-id)
(deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :transform,
:from_entity_id transform-id,
@@ -390,10 +382,12 @@
(t2/delete! :model/Database db-id)
(is (nil? (t2/select-one :model/Database db-id))
"the Database row is actually gone")
+ (assert-stale :transform transform-id)
+ (deps.test/synchronously-run-backfill!)
(is (true? (t2/select-one-fn :stale :model/AnalysisFinding
:analyzed_entity_type :transform
:analyzed_entity_id transform-id))
- "after db-delete the transform's finding is marked stale")
+ "after db-delete the backfill marks the transform's finding stale")
(testing "and re-running the entity-check job flips :result false so the transform shows on broken-diagnostics"
(#'task.entity-check/check-entities!)
(let [{:keys [stale result]} (t2/select-one :model/AnalysisFinding
@@ -416,12 +410,7 @@
(let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
(mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
:model/Transform {transform-id :id} {:target target}]
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
+ (deps.hooks/after-transform-run transform-id table-id)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
@@ -429,7 +418,7 @@
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))
(deftest ^:synchronized python-transform-update-handles-downstream-dependencies-test
- (testing "python transform update events handles downstream dependencies"
+ (testing "python transform updates handle downstream dependencies"
(run-with-dependencies-setup!
(fn [_]
(let [source {:type :python,
@@ -444,24 +433,18 @@
target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
(mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}
- :model/Transform {transform-id :id :as transform} {:target target
- :source source}]
+ :model/Transform {transform-id :id} {:target target
+ :source source}]
(testing "initial run"
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
+ (deps.hooks/after-transform-run transform-id table-id)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
:to_entity_id transform-id}]
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "keeping target"
- (events/publish-event! :event/update-transform
- {:object transform
- :user-id api/*current-user-id*})
+ (t2/update! :model/Transform transform-id {:source (update source :body str "\n")})
+ (assert-stale :transform transform-id)
(deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
@@ -470,16 +453,12 @@
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "changing target update"
(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
- (let [updated (t2/select-one :model/Transform :id transform-id)]
- (events/publish-event! :event/update-transform
- {:object updated
- :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
+ (deps.test/synchronously-run-backfill!)
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
(deftest ^:synchronized query-transform-update-handles-downstream-dependencies-test
- (testing "query transform update events handles downstream dependencies"
+ (testing "query transform updates handle downstream dependencies"
(run-with-dependencies-setup!
(fn [mp]
(let [source {:query (lib/native-query mp "select * from orders")
@@ -487,23 +466,17 @@
target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}]
(mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}
:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}
- :model/Transform {transform-id :id :as transform} {:target target :source source}]
+ :model/Transform {transform-id :id} {:target target :source source}]
(testing "initial run"
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "Other"
- :output-table :test_table
- :transform-id transform-id}
- :user-id api/*current-user-id*})
+ (deps.hooks/after-transform-run transform-id table-id)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
:to_entity_type :transform,
:to_entity_id transform-id}]
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "keeping target"
- (events/publish-event! :event/update-transform
- {:object transform
- :user-id api/*current-user-id*})
+ (t2/update! :model/Transform transform-id {:source (assoc source :query (lib/native-query mp "select * from orders limit 10"))})
+ (assert-stale :transform transform-id)
(deps.test/synchronously-run-backfill!)
(is (=? [{:from_entity_type :table
:from_entity_id table-id,
@@ -512,13 +485,9 @@
(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))
(testing "changing target update"
(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})
- (let [updated (t2/select-one :model/Transform :id transform-id)]
- (events/publish-event! :event/update-transform
- {:object updated
- :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (empty?
- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))
+ (deps.test/synchronously-run-backfill!)
+ (is (empty?
+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))
(deftest segment-update-sets-correct-dependencies
(run-with-dependencies-setup!
@@ -526,11 +495,10 @@
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)
category-field-id (mt/id :products :category)]
- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id
- :definition {:filter [:> [:field price-field-id nil] 50]}}]
+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 50]}}]
(testing "creating a segment creates dependency to its table"
- (events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})
- (assert-stale :segment segment-id)
+ (assert-pending :segment segment-id)
(deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :segment
:from_entity_id segment-id
@@ -540,18 +508,17 @@
(t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))
(testing "updating segment definition recalculates dependencies"
(t2/update! :model/Segment segment-id {:definition {:filter [:= [:field category-field-id nil] "Widget"]}})
- (let [updated-segment (t2/select-one :model/Segment :id segment-id)]
- (events/publish-event! :event/segment-update {:object updated-segment :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (= #{{:from_entity_type :segment
- :from_entity_id segment-id
- :to_entity_type :table
- :to_entity_id products-id}}
- (into #{} (map #(dissoc % :id)
- (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))
+ (assert-stale :segment segment-id)
+ (deps.test/synchronously-run-backfill!)
+ (is (= #{{:from_entity_type :segment
+ :from_entity_id segment-id
+ :to_entity_type :table
+ :to_entity_id products-id}}
+ (into #{} (map #(dissoc % :id)
+ (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))
(testing "deleting segment removes all dependencies"
(t2/delete! :model/Segment segment-id)
- (events/publish-event! :event/segment-delete {:object segment :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))))
(deftest card-with-segment-dependencies
@@ -559,16 +526,14 @@
(fn [_]
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)]
- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id
- :definition {:filter [:> [:field price-field-id nil] 50]}}]
+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 50]}}]
(testing "creating a card using a segment creates dependencies to both segment and table"
- (events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})
(deps.test/synchronously-run-backfill!)
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query {:database (mt/id)
- :type :query
- :query {:source-table products-id
- :filter [:segment segment-id]}}}]
- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query {:database (mt/id)
+ :type :query
+ :query {:source-table products-id
+ :filter [:segment segment-id]}}}]
(deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :card
:from_entity_id card-id
@@ -589,12 +554,11 @@
orders (lib.metadata/table mp orders-id)
quantity (lib.metadata/field mp (mt/id :orders :quantity))
subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]
- (mt/with-temp [:model/Measure {measure-id :id :as measure} {:table_id orders-id
- :definition (-> (lib/query mp orders)
- (lib/aggregate (lib/sum quantity)))}]
+ (mt/with-temp [:model/Measure {measure-id :id} {:table_id orders-id
+ :definition (-> (lib/query mp orders)
+ (lib/aggregate (lib/sum quantity)))}]
(testing "creating a measure creates dependency to its table"
- (events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})
- (assert-stale :measure measure-id)
+ (assert-pending :measure measure-id)
(deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure-id
@@ -605,18 +569,17 @@
(testing "updating measure definition recalculates dependencies"
(t2/update! :model/Measure measure-id {:definition (-> (lib/query mp orders)
(lib/aggregate (lib/sum subtotal)))})
- (let [updated-measure (t2/select-one :model/Measure :id measure-id)]
- (events/publish-event! :event/measure-update {:object updated-measure :user-id api/*current-user-id*})
- (deps.test/synchronously-run-backfill!)
- (is (= #{{:from_entity_type :measure
- :from_entity_id measure-id
- :to_entity_type :table
- :to_entity_id orders-id}}
- (into #{} (map #(dissoc % :id)
- (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure)))))))
+ (assert-stale :measure measure-id)
+ (deps.test/synchronously-run-backfill!)
+ (is (= #{{:from_entity_type :measure
+ :from_entity_id measure-id
+ :to_entity_type :table
+ :to_entity_id orders-id}}
+ (into #{} (map #(dissoc % :id)
+ (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))
(testing "deleting measure removes all dependencies"
(t2/delete! :model/Measure measure-id)
- (events/publish-event! :event/measure-delete {:object measure :user-id api/*current-user-id*})
+ (deps.test/synchronously-run-backfill!)
(is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))
(deftest measure-with-measure-dependencies
@@ -626,17 +589,15 @@
orders (lib.metadata/table mp orders-id)
quantity (lib.metadata/field mp (mt/id :orders :quantity))
subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]
- (mt/with-temp [:model/Measure {measure1-id :id :as measure1} {:table_id orders-id
- :definition (-> (lib/query mp orders)
- (lib/aggregate (lib/sum quantity)))}]
+ (mt/with-temp [:model/Measure {measure1-id :id} {:table_id orders-id
+ :definition (-> (lib/query mp orders)
+ (lib/aggregate (lib/sum quantity)))}]
(let [mp' (mt/metadata-provider)]
- (mt/with-temp [:model/Measure {measure2-id :id :as measure2} {:table_id orders-id
- :definition (-> (lib/query mp' orders)
- (lib/aggregate (lib/+ (lib.metadata/measure mp' measure1-id)
- (lib/sum subtotal))))}]
+ (mt/with-temp [:model/Measure {measure2-id :id} {:table_id orders-id
+ :definition (-> (lib/query mp' orders)
+ (lib/aggregate (lib/+ (lib.metadata/measure mp' measure1-id)
+ (lib/sum subtotal))))}]
(testing "creating a measure that references another measure creates dependencies to both"
- (events/publish-event! :event/measure-create {:object measure1 :user-id api/*current-user-id*})
- (events/publish-event! :event/measure-create {:object measure2 :user-id api/*current-user-id*})
(deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :measure
:from_entity_id measure2-id
@@ -655,17 +616,15 @@
(let [orders-id (mt/id :orders)
orders (lib.metadata/table mp orders-id)
quantity (lib.metadata/field mp (mt/id :orders :quantity))]
- (mt/with-temp [:model/Measure {measure-id :id :as measure} {:table_id orders-id
- :definition (-> (lib/query mp orders)
- (lib/aggregate (lib/sum quantity)))}]
- (events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})
+ (mt/with-temp [:model/Measure {measure-id :id} {:table_id orders-id
+ :definition (-> (lib/query mp orders)
+ (lib/aggregate (lib/sum quantity)))}]
(deps.test/synchronously-run-backfill!)
(testing "creating a card using a measure creates dependencies to both measure and table"
(let [mp' (mt/metadata-provider)
query (-> (lib/query mp' orders)
(lib/aggregate (lib.metadata/measure mp' measure-id)))]
- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query query}]
- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}]
(deps.test/synchronously-run-backfill!)
(is (= #{{:from_entity_type :card
:from_entity_id card-id
@@ -705,13 +664,13 @@
(testing "Card updates should mark the card stale and trigger the background job"
(let [products-id (mt/id :products)
products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}
+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}
:model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}
:model/Dependency _ {:from_entity_type :card
:from_entity_id child-card-id
:to_entity_type :card
:to_entity_id parent-card-id}]
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))})
;; No analysis record should exist yet - mark-entity-stale! is a no-op for never-analyzed entities
(assert-has-analyses
{:card {parent-card-id nil}})
@@ -739,17 +698,23 @@
(deps.findings/upsert-analysis! parent-card)
(deps.findings/upsert-analysis! child-card)
(deps.findings/upsert-analysis! other-card)
+ (deps.dependency-status/upsert-status! :card child-card-id)
+ (deps.dependency-status/upsert-status! :card other-card-id)
(testing "Checking that the initial analyses exist"
(assert-has-analyses
{:card {parent-card-id old-version
child-card-id old-version
other-card-id old-version}}))
- (testing "card update only marks direct dependents stale - not its own analysis"
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ (testing "changing the card query marks its finding and its dependents' stale, not unrelated ones"
+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})
+ (deps.test/synchronously-run-backfill!)
(assert-has-analyses
{:card {parent-card-id old-version
child-card-id -1
- other-card-id old-version}}))
+ other-card-id old-version}})
+ (is (= {parent-card-id true child-card-id true other-card-id false}
+ (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding :analyzed_entity_type :card
+ :analyzed_entity_id [:in [parent-card-id child-card-id other-card-id]]))))
(testing "If the version changes, the job picks up only the stale cards"
(binding [models.analysis-finding/*current-analysis-finding-version* (inc old-version)]
(assert-has-analyses
@@ -769,8 +734,9 @@
:to_entity_type :card :to_entity_id parent-id}]
(deps.findings/upsert-analysis! parent)
(deps.findings/upsert-analysis! child)
- ;; Event marks entity stale in analysis_finding
- (events/publish-event! :event/card-update {:object parent :previous-object parent :user-id api/*current-user-id*})
+ (deps.dependency-status/upsert-status! :card child-id)
+ (t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")})
+ (deps.test/synchronously-run-backfill!)
(testing "Parent card should be marked stale"
(is (true? (t2/select-one-fn :stale :model/AnalysisFinding
:analyzed_entity_type :card
@@ -811,8 +777,7 @@
(assert-has-analyses
{:card {parent-card-id old-version child-card-id old-version}
:transform {transform-id old-version}})
- ;; Event marks parent stale in analysis_finding, which triggers entity-check
- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})
+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})
;; Run entity-check job to drain the stale entities
(#'task.entity-check/check-entities!)
(testing "All entities should be re-analyzed after the drain"
@@ -828,14 +793,15 @@
(fn [mp]
(testing "Transform update does not error when no analyses exist yet"
(let [products (lib.metadata/table mp (mt/id :products))]
- (mt/with-temp [:model/Transform transform {:source {:type :query
- :query (lib/query mp products)}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}]
- ;; Should not throw even when no analysis findings exist
- (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})))))))))
+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
+ :query (lib/query mp products)}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ (is (= 1 (t2/update! :model/Transform transform-id
+ {:source {:type :query
+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})))))))))
(deftest ^:synchronized transform-update-updates-analyses-test
(run-with-dependencies-setup!
@@ -851,7 +817,9 @@
:type :table}}]
(deps.findings/upsert-analysis! transform)
(assert-has-analyses {:transform {transform-id old-version}})
- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})
+ (t2/update! :model/Transform transform-id {:source {:type :query
+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})
+ (deps.test/synchronously-run-backfill!)
(testing "Transform should be marked stale"
(is (true? (t2/select-one-fn :stale :model/AnalysisFinding
:analyzed_entity_type :transform
@@ -866,14 +834,15 @@
(run-with-dependencies-setup!
(fn [mp]
(testing "Native transform updates do not error"
- (mt/with-temp [:model/Transform transform {:source {:type :query
- :query (lib/native-query mp "select * from products")}
- :name "transform_sample"
- :target {:schema "public"
- :name "sample"
- :type :table}}]
- ;; Should not throw even for native transforms
- (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*}))))))))
+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query
+ :query (lib/native-query mp "select * from products")}
+ :name "transform_sample"
+ :target {:schema "public"
+ :name "sample"
+ :type :table}}]
+ (is (= 1 (t2/update! :model/Transform transform-id
+ {:source {:type :query
+ :query (lib/native-query mp "select * from orders")}}))))))))
(deftest ^:synchronized transform-run-works-with-no-analyses-test
(run-with-dependencies-setup!
@@ -886,12 +855,7 @@
:target {:schema "public" :name "sample" :type :table}}
:model/Dependency _ {:from_entity_type :card :from_entity_id card-id
:to_entity_type :transform :to_entity_id transform-id}]
- (is (some? (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "public"
- :output-table "sample"
- :transform-id transform-id}
- :user-id api/*current-user-id*})))))))))
+ (is (nil? (deps.hooks/after-transform-run transform-id nil)))))))))
(deftest ^:synchronized transform-run-marks-dependents-stale-test
(run-with-dependencies-setup!
@@ -909,12 +873,7 @@
(deps.findings/upsert-analysis! card)
(deps.findings/upsert-analysis! transform)
(deps.findings/upsert-analysis! other-card)
- (events/publish-event! :event/transform-run-complete
- {:object {:db-id (mt/id)
- :output-schema "public"
- :output-table "sample"
- :transform-id transform-id}
- :user-id api/*current-user-id*})
+ (deps.hooks/after-transform-run transform-id nil)
(testing "Dependent card should be marked stale"
(is (true? (t2/select-one-fn :stale :model/AnalysisFinding
:analyzed_entity_type :card :analyzed_entity_id card-id))))
@@ -931,10 +890,10 @@
(testing "Segment update does not error when no analyses exist yet"
(let [products-id (mt/id :products)
price-field-id (mt/id :products :price)]
- (mt/with-temp [:model/Segment segment {:table_id products-id
- :definition {:filter [:> [:field price-field-id nil] 50]}}]
- ;; Should not throw even when no analysis findings exist
- (is (some? (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})))))))))
+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 50]}}]
+ (is (= 1 (t2/update! :model/Segment segment-id
+ {:definition {:filter [:< [:field price-field-id nil] 50]}})))))))))
(deftest ^:synchronized segment-update-updates-analyses-test
(run-with-dependencies-setup!
@@ -951,8 +910,10 @@
:to_entity_type :segment :to_entity_id segment-id}]
(deps.findings/upsert-analysis! segment)
(deps.findings/upsert-analysis! card)
+ (deps.dependency-status/upsert-status! :card card-id)
(assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})
- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})
+ (t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})
+ (deps.test/synchronously-run-backfill!)
(testing "Segment should be marked stale"
(is (true? (t2/select-one-fn :stale :model/AnalysisFinding
:analyzed_entity_type :segment :analyzed_entity_id segment-id))))
@@ -978,14 +939,14 @@
(deps.findings/upsert-analysis! card)
(deps.findings/upsert-analysis! other-card)
(assert-has-analyses {:card {card-id old-version other-card-id old-version}})
- (let [table (t2/select-one :model/Table :id products-id)]
- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})
- (testing "Card depending on updated table should be marked stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id card-id))))
- (testing "Card not depending on updated table should not be stale"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))
+ (deps.test/do-with-immediate-hook-writes
+ #(mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]
+ (testing "Card depending on updated table should be marked stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id card-id))))
+ (testing "Card not depending on updated table should not be stale"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))
(deftest ^:synchronized table-metadata-update-works-with-no-analyses-test
(run-with-dependencies-setup!
@@ -996,8 +957,7 @@
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
:model/Dependency _ {:from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id products-id}]
- (let [table (t2/select-one :model/Table :id products-id)]
- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})
+ (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]
(assert-has-analyses {:card {card-id nil}}))))))))
(deftest ^:synchronized field-metadata-update-triggers-dependent-analysis-test
@@ -1015,14 +975,14 @@
(deps.findings/upsert-analysis! card)
(deps.findings/upsert-analysis! other-card)
(assert-has-analyses {:card {card-id old-version other-card-id old-version}})
- (let [field (t2/select-one :model/Field :id (mt/id :products :category))]
- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})
- (testing "Card depending on the table whose field was updated should be stale"
- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id card-id))))
- (testing "Card not depending on that table should not be stale"
- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
- :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))
+ (deps.test/do-with-immediate-hook-writes
+ #(mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]
+ (testing "Card depending on the table whose field was updated should be stale"
+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id card-id))))
+ (testing "Card not depending on that table should not be stale"
+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))
(deftest ^:synchronized field-metadata-update-works-with-no-analyses-test
(run-with-dependencies-setup!
@@ -1033,6 +993,32 @@
(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}
:model/Dependency _ {:from_entity_type :card :from_entity_id card-id
:to_entity_type :table :to_entity_id products-id}]
- (let [field (t2/select-one :model/Field :id (mt/id :products :category))]
- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})
+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]
(assert-has-analyses {:card {card-id nil}}))))))))
+
+(deftest serdes-load-recalculates-changed-entities-test
+ (testing "a serdes load recalculates the dependencies of the entities it changes, and only those"
+ (run-with-dependencies-setup!
+ (fn [mp]
+ (let [card-deps #(t2/select-fn-set (juxt :to_entity_type :to_entity_id)
+ :model/Dependency :from_entity_type :card :from_entity_id %)
+ query #(lib/query mp (lib.metadata/table mp (mt/id %)))
+ processed #(t2/exists? :model/DependencyStatus :entity_type :card :entity_id % :stale false)
+ load! #(serdes/with-cache
+ (serdes.load/load-metabase! (ts/ingestion-in-memory %) :reindex? false))]
+ (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (query :products)}
+ :model/Card {other-id :id} {:dataset_query (query :products)}]
+ (let [exported (serdes/extract-one "Card" {} card)]
+ (t2/update! :model/Card card-id {:dataset_query (query :orders)})
+ (deps.dependency-status/mark-stale! {:card #{card-id other-id}})
+ (deps.test/synchronously-run-backfill!)
+ (is (= #{[:table (mt/id :orders)]} (card-deps card-id)))
+ (load! [exported])
+ (testing "only the changed card is marked stale"
+ (assert-stale :card card-id)
+ (is (processed other-id)))
+ (deps.test/synchronously-run-backfill!)
+ (is (= #{[:table (mt/id :products)]} (card-deps card-id)))
+ (testing "reloading it unchanged marks nothing"
+ (load! [exported])
+ (is (processed card-id))))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
index c8623687cfe..9dee6dc38fb 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj
@@ -330,7 +330,7 @@
(let [cid (:id card)]
;; seed a clean finding, mark it stale, then make its database unresolvable
(models.analysis-finding/upsert-analysis! :card cid true [])
- (models.analysis-finding/mark-stale! :card [cid])
+ (models.analysis-finding/mark-stale! {:card #{cid}})
(with-redefs-fn {#'deps.findings/instance-db-id (constantly nil)}
(fn []
(lib-be/with-metadata-provider-cache
@@ -363,7 +363,7 @@
:analyzed_entity_type :card :analyzed_entity_id cid))
"phase 1: stale flag cleared")
;; phase 2: database resolvable again; re-trigger analysis (as an event/job would)
- (models.analysis-finding/mark-stale! :card [cid])
+ (models.analysis-finding/mark-stale! {:card #{cid}})
(lib-be/with-metadata-provider-cache
(deps.findings/upsert-analysis! (t2/instance :model/Card card)))
(is (empty? (models.analysis-finding-error/errors-for-entity :card cid))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
new file mode 100644
index 00000000000..d3e8dec01e6
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj
@@ -0,0 +1,190 @@
+(ns metabase-enterprise.dependencies.hooks-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.dependencies.hooks]
+ [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
+ [metabase-enterprise.dependencies.task.backfill :as task.backfill]
+ [metabase-enterprise.dependencies.test-util :as deps.test]
+ [metabase-enterprise.serialization.test-util :as ts]
+ [metabase-enterprise.serialization.v2.load :as serdes.load]
+ [metabase.batch-processing.core :as grouper]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.models.serialization :as serdes]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(comment
+ metabase-enterprise.dependencies.hooks/keep-me)
+
+(defn- stale? [entity-type entity-id]
+ (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true))
+
+(defn- table-query [table]
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/table mp (mt/id table)))))
+
+(defn- card-query [card-id]
+ (let [mp (mt/metadata-provider)]
+ (lib/query mp (lib.metadata/card mp card-id))))
+
+(deftest card-update-test
+ (testing "updating a card marks it stale only when a column its dependencies come from changes"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/DependencyStatus]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
+ (deps.dependency-status/upsert-status! :card card-id)
+ (t2/update! :model/Card card-id {:name "Renamed"})
+ (is (not (stale? :card card-id)))
+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
+ (is (stale? :card card-id)))))))
+
+(deftest card-update-marks-finding-test
+ (testing "updating a card's query marks its own analysis finding stale in the same transaction"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/DependencyStatus :model/AnalysisFinding]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
+ (deps.analysis-finding/upsert-analysis! :card card-id true [])
+ (let [finding-stale? #(t2/select-one-fn :stale :model/AnalysisFinding
+ :analyzed_entity_type :card :analyzed_entity_id card-id)]
+ (t2/update! :model/Card card-id {:name "Renamed"})
+ (is (not (finding-stale?)))
+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
+ (is (finding-stale?))))))))
+
+(deftest card-update-without-feature-test
+ (testing "without the dependencies feature the hooks do nothing"
+ (mt/with-premium-features #{}
+ (mt/with-model-cleanup [:model/DependencyStatus]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
+ (deps.dependency-status/upsert-status! :card card-id)
+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})
+ (is (not (stale? :card card-id))))))))
+
+(deftest dashboard-card-test
+ (testing "inserting, updating and deleting a dashboard card, or adding a series, marks its dashboard stale"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/DependencyStatus]
+ (mt/with-temp [:model/Dashboard {dashboard-id :id} {}
+ :model/Card {card-id :id} {:dataset_query (table-query :products)}
+ :model/Card {other-id :id} {:dataset_query (table-query :orders)}]
+ (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id)]
+ (processed!)
+ (let [dashcard-id (t2/insert-returning-pk! :model/DashboardCard
+ {:dashboard_id dashboard-id :card_id card-id
+ :row 0 :col 0 :size_x 4 :size_y 4})]
+ (testing "inserting a dashboard card"
+ (is (stale? :dashboard dashboard-id)))
+ (processed!)
+ (t2/update! :model/DashboardCard dashcard-id {:row 1})
+ (testing "moving a dashboard card"
+ (is (not (stale? :dashboard dashboard-id))))
+ (t2/update! :model/DashboardCard dashcard-id {:card_id other-id})
+ (testing "pointing a dashboard card at another card"
+ (is (stale? :dashboard dashboard-id)))
+ (processed!)
+ (t2/insert! :model/DashboardCardSeries {:dashboardcard_id dashcard-id :card_id card-id :position 0})
+ (testing "adding a series"
+ (is (stale? :dashboard dashboard-id)))
+ (processed!)
+ (t2/delete! :model/DashboardCard :id dashcard-id)
+ (testing "removing a dashboard card"
+ (is (stale? :dashboard dashboard-id))))))))))
+
+(deftest entity-delete-test
+ (testing "deleting an entity marks its status stale, and the backfill drops its dependencies and status"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]
+ (t2/insert! :model/Dependency {:from_entity_type "card" :from_entity_id card-id
+ :to_entity_type "table" :to_entity_id (mt/id :products)})
+ (deps.dependency-status/upsert-status! :card card-id)
+ (t2/delete! :model/Card :id card-id)
+ (is (stale? :card card-id))
+ (deps.test/synchronously-run-backfill!)
+ (is (not (t2/exists? :model/Dependency :from_entity_type "card" :from_entity_id card-id)))
+ (is (not (t2/exists? :model/DependencyStatus :entity_type "card" :entity_id card-id)))))))
+
+(deftest concurrent-saves-of-dependent-cards-commit-test
+ (testing "saving a card and then its dependent in one transaction, while another request saves the dependent, commits both"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/DependencyStatus]
+ (let [insert-card! (fn [query]
+ (t2/insert-returning-pk! :model/Card {:name (mt/random-name)
+ :display :table
+ :visualization_settings {}
+ :creator_id (mt/user->id :crowberto)
+ :database_id (mt/id)
+ :dataset_query query}))
+ card-a (insert-card! (table-query :products))
+ card-b (insert-card! (card-query card-a))
+ a-saved (promise)
+ b-status-marked (promise)
+ save! (fn [thunk]
+ (future
+ (try
+ (thunk)
+ :committed
+ (catch Throwable e
+ (ex-message (or (ex-cause e) e))))))
+ mark-stale! (mt/original-fn #'deps.dependency-status/mark-stale!)]
+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-b
+ :to_entity_type :card :to_entity_id card-a})
+ (doseq [card-id [card-a card-b]]
+ (deps.dependency-status/upsert-status! :card card-id)
+ (deps.analysis-finding/upsert-analysis! :card card-id true []))
+ (mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!
+ (fn [type->ids]
+ (mark-stale! type->ids)
+ (when (= {:card #{card-b}} type->ids)
+ (deliver b-status-marked true)))]
+ (let [cascade (save! #(t2/with-transaction [_conn]
+ (t2/update! :model/Card card-a {:dataset_query (table-query :orders)})
+ (deliver a-saved true)
+ (deref b-status-marked 10000 nil)
+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 5)})))
+ single (save! #(do (deref a-saved 10000 nil)
+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 11)})))]
+ (is (= [:committed :committed]
+ [(deref cascade 30000 :timeout) (deref single 30000 :timeout)])))))))))
+
+(deftest serdes-load-triggers-jobs-once-test
+ (testing "a serdes load marks what it writes, nested deletes included, and triggers the jobs once it finishes"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/DependencyStatus]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}
+ :model/Dashboard {dashboard-id :id} {}
+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id}]
+ (let [exported (ts/extract-one "Dashboard" dashboard-id)
+ submitted (atom [])
+ triggered (atom 0)]
+ (t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id
+ :row 4 :col 0 :size_x 4 :size_y 4})
+ (deps.dependency-status/upsert-status! :dashboard dashboard-id)
+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))
+ task.backfill/trigger-backfill-job! (fn [] (swap! triggered inc))]
+ (deps.test/do-with-immediate-hook-writes
+ #(serdes/with-cache
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))
+ (testing "deleting the dashboard card missing from the export marks its dashboard"
+ (is (= 1 (t2/count :model/DashboardCard :dashboard_id dashboard-id)))
+ (is (stale? :dashboard dashboard-id)))
+ (testing "the load triggers the jobs once, after it finishes"
+ (is (= [] @submitted))
+ (is (= 1 @triggered)))))))))
+
+(deftest serdes-reload-of-unchanged-dashboard-test
+ (testing "reloading an unchanged dashboard export, including empty visualization settings, does not mark it"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/DependencyStatus]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}
+ :model/Dashboard {dashboard-id :id} {}
+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id
+ :visualization_settings {}}]
+ (let [exported (ts/extract-one "Dashboard" dashboard-id)]
+ (deps.dependency-status/upsert-status! :dashboard dashboard-id)
+ (mt/with-dynamic-fn-redefs [task.backfill/trigger-backfill-job! (fn [])]
+ (deps.test/do-with-immediate-hook-writes
+ #(serdes/with-cache
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))
+ (is (not (stale? :dashboard dashboard-id)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
deleted file mode 100644
index a6a9f1e5ce3..00000000000
--- a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj
+++ /dev/null
@@ -1,419 +0,0 @@
-(ns metabase-enterprise.dependencies.metadata-update-test
- (:require
- [clojure.java.jdbc :as jdbc]
- [clojure.test :refer [deftest is testing]]
- [java-time.api :as t]
- [metabase-enterprise.dependencies.async :as async]
- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
- [metabase-enterprise.dependencies.events :as deps.events]
- [metabase-enterprise.dependencies.findings :as deps.findings]
- [metabase-enterprise.dependencies.metadata-update :as deps.metadata-update]
- [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]
- [metabase.api.common :as api]
- [metabase.events.core :as events]
- [metabase.lib-be.core :as lib-be]
- [metabase.lib.core :as lib]
- [metabase.lib.metadata :as lib.metadata]
- [metabase.queries.card-schema :as card-schema]
- [metabase.sync.core :as sync]
- [metabase.test :as mt]
- [metabase.test.data.one-off-dbs :as one-off-dbs]
- [metabase.util :as u]
- [toucan2.core :as t2]))
-
-(defn assert-traverses-grandchild [{:keys [grandchild-dependency-type grandchild-data should-traverse? should-include?]}]
- (let [mp (mt/metadata-provider)
- query (->> (mt/id :products)
- (lib.metadata/table mp)
- (lib/query mp))
- grandchild-model (deps.dependency-types/dependency-type->model grandchild-dependency-type)]
- (mt/with-temp [:model/Card {parent-id :id} {:dataset_query query}
- :model/Card {child-id :id} {:dataset_query query}
- grandchild-model {grandchild-id :id} grandchild-data
- :model/Card {great-grandchild-id :id} {:dataset_query query}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}
- :model/Dependency _ {:from_entity_type grandchild-dependency-type
- :from_entity_id grandchild-id
- :to_entity_type :card
- :to_entity_id child-id}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id great-grandchild-id
- :to_entity_type grandchild-dependency-type
- :to_entity_id grandchild-id}]
- (is (= (cond-> #{child-id}
- should-include? (conj grandchild-id)
- should-traverse? (conj great-grandchild-id))
- (set (#'deps.metadata-update/dependent-mbql-cards
- (#'deps.metadata-update/mbql-graph mp)
- :card parent-id)))))))
-
-(deftest ^:parallel dependent-mbql-cards-traverses-cards-test
- (testing "dependent-mbql-cards looks through card dependencies"
- (let [mp (mt/metadata-provider)
- query (->> (mt/id :products)
- (lib.metadata/table mp)
- (lib/query mp))]
- (assert-traverses-grandchild
- {:grandchild-dependency-type :card
- :grandchild-data {:dataset_query query}
- :should-traverse? true
- :should-include? true}))))
-
-(deftest ^:parallel dependent-mbql-cards-stops-on-native-cards-test
- (testing "dependent-mbql-cards stops when it reaches a native card"
- (let [mp (mt/metadata-provider)
- native-query (lib/native-query mp "select * from orders")]
- (assert-traverses-grandchild
- {:grandchild-dependency-type :card
- :grandchild-data {:dataset_query native-query}}))))
-
-(deftest ^:parallel dependent-mbql-cards-stops-on-snippets-test
- (testing "dependent-mbql-cards stops when it reaches a snippet"
- (assert-traverses-grandchild
- {:grandchild-dependency-type :snippet
- :grandchild-data {:name "test snippet"
- :content "SELECT 1"}})))
-
-(deftest ^:parallel dependent-mbql-cards-stops-on-transforms-test
- (testing "dependent-mbql-cards stops when it reaches a transform"
- (assert-traverses-grandchild
- {:grandchild-dependency-type :transform
- :grandchild-data {}})))
-
-(deftest ^:parallel dependent-mbql-cards-stops-on-dashboards-test
- (testing "dependent-mbql-cards stops when it reaches a dashboard"
- (assert-traverses-grandchild
- {:grandchild-dependency-type :dashboard
- :grandchild-data {}})))
-
-(deftest ^:parallel dependent-mbql-cards-stops-on-documents-test
- (testing "dependent-mbql-cards stops when it reaches a document"
- (assert-traverses-grandchild
- {:grandchild-dependency-type :document
- :grandchild-data {}})))
-
-(deftest ^:parallel dependent-mbql-cards-traverses-through-segments-test
- (testing "dependent-mbql-cards traverses through segments"
- (assert-traverses-grandchild
- {:grandchild-dependency-type :segment
- :grandchild-data {}
- :should-traverse? true})))
-
-(deftest ^:parallel dependent-mbql-cards-traverses-through-measures-test
- (testing "dependent-mbql-cards traverses through measures"
- (assert-traverses-grandchild
- {:grandchild-dependency-type :measure
- :grandchild-data {}
- :should-traverse? true})))
-
-(deftest ^:synchronized card-update-updates-child-metadata-test
- (testing "card updates update child card metadata"
- (mt/with-test-user :rasta
- (mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)
- products-id (mt/id :products)
- orders-id (mt/id :orders)
- products (lib.metadata/table mp products-id)
- orders (lib.metadata/table mp orders-id)]
- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)}
- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id grandchild-id
- :to_entity_type :card
- :to_entity_id child-id}]
- (is (= #{8}
- (t2/select-fn-set (comp count :result_metadata)
- (card-schema/selection [:database_id])
- :id [:in [parent-id child-id grandchild-id]])))
- (t2/update! :model/Card parent-id {:dataset_query (lib/query mp orders)})
- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update
- {:object (assoc parent-card :dataset_query (lib/query mp orders))
- :previous-object parent-card
- :user-id api/*current-user-id*}))
- (is (= #{9}
- (t2/select-fn-set (comp count :result_metadata)
- (card-schema/selection [:database_id])
- :id [:in [parent-id child-id grandchild-id]])))))))))
-
-(deftest ^:synchronized native-card-update-does-not-update-children-test
- (testing "native card updates do not update children"
- (mt/with-test-user :rasta
- (mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)
- products-id (mt/id :products)
- products (lib.metadata/table mp products-id)
- native-query (lib/native-query mp "select * from orders")]
- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)}
- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}]
- (t2/update! :model/Card child-id {:result_metadata nil})
- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update
- {:object (assoc parent-card :dataset_query native-query)
- :previous-object parent-card
- :user-id api/*current-user-id*}))
- (is (= nil
- (t2/select-one-fn :result_metadata
- (card-schema/selection [:database_id])
- :id child-id)))))))))
-
-(deftest ^:synchronized model-update-passes-down-new-values-test
- (testing "model updates pass down new result metadata"
- (mt/with-test-user :rasta
- (mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)
- products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)
- :type :model}
- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))
-
- :type :model}
- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))
- :type :model}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id grandchild-id
- :to_entity_type :card
- :to_entity_id child-id}]
- (let [new-result-metadata (assoc-in (:result_metadata parent-card)
- [0 :display_name]
- "new-name")]
- (t2/update! :model/Card parent-id {:result_metadata new-result-metadata})
- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update
- {:object (assoc parent-card :result_metadata new-result-metadata)
- :previous-object parent-card
- :user-id api/*current-user-id*}))
- (is (= #{[child-id "new-name"] [grandchild-id "new-name"]}
- (t2/select-fn-set (juxt :id #(get-in % [:result_metadata 0 :display_name]))
- (card-schema/selection [:database_id])
- :id [:in [child-id grandchild-id]]))))))))))
-
-(deftest ^:synchronized model-update-respects-child-overrides-test
- (testing "model updates respect child metadata edits"
- (mt/with-test-user :rasta
- (mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)
- products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)
- :type :model}
- :model/Card {child-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))
- :type :model}
- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))
- :type :model}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id grandchild-id
- :to_entity_type :card
- :to_entity_id child-id}]
- (let [new-parent-metadata (assoc-in (:result_metadata parent-card)
- [0 :display_name]
- "new-name")
- new-child-metadata (assoc-in (:result_metadata child-card)
- [0 :display_name]
- "child-name")
- new-grandchild-metadata (assoc-in (:result_metadata child-card)
- [0 :display_name]
- "grandchild-name")]
- (t2/update! :model/Card parent-id {:result_metadata new-parent-metadata})
- (t2/update! :model/Card child-id {:result_metadata new-child-metadata})
- (t2/update! :model/Card grandchild-id {:result_metadata new-grandchild-metadata})
- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update
- {:object (assoc parent-card :result_metadata new-parent-metadata)
- :previous-object parent-card
- :user-id api/*current-user-id*}))
- (is (= #{[child-id "child-name"] [grandchild-id "grandchild-name"]}
- (t2/select-fn-set (juxt :id #(get-in % [:result_metadata 0 :display_name]))
- (card-schema/selection [:database_id])
- :id [:in [child-id grandchild-id]]))))))))))
-
-(deftest ^:synchronized model-update-stops-recursing-when-child-metadata-is-unchanged-test
- (testing "model updates stop recursing when they hit a child whose metadata didn't change"
- (mt/with-test-user :rasta
- (mt/with-premium-features #{:dependencies}
- (let [mp (mt/metadata-provider)
- products-id (mt/id :products)
- products (lib.metadata/table mp products-id)]
- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)
- :type :model}
- :model/Card {child-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))
- :type :model}
- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))
- :type :model}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id child-id
- :to_entity_type :card
- :to_entity_id parent-id}
- :model/Dependency _ {:from_entity_type :card
- :from_entity_id grandchild-id
- :to_entity_type :card
- :to_entity_id child-id}]
- (let [new-parent-metadata (assoc-in (:result_metadata parent-card)
- [0 :display_name]
- "new-name")
- new-child-metadata (-> (:result_metadata child-card)
- (assoc-in [0 :display_name]
- "child-name")
- (assoc-in [0 :lib/original-display-name]
- "new-name"))]
- (t2/update! :model/Card parent-id {:result_metadata new-parent-metadata})
- (t2/update! :model/Card child-id {:result_metadata new-child-metadata})
- (t2/update! :model/Card grandchild-id {:result_metadata nil})
- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]
- (events/publish-event! :event/card-update
- {:object (assoc parent-card :result_metadata new-parent-metadata)
- :previous-object parent-card
- :user-id api/*current-user-id*}))
- (is (= nil
- (t2/select-one-fn #(get-in % [:result_metadata 0 :display_name])
- (card-schema/selection [:database_id])
- :id grandchild-id))))))))))
-
-(defn- with-syncable-db! [thunk]
- (mt/with-premium-features #{:dependencies}
- (one-off-dbs/with-blank-db
- ;; Step 1: Create a table with two columns
- (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"
- "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]
- (jdbc/execute! one-off-dbs/*conn* [statement]))
- ;; Sync the database to pick up the new table
- (sync/sync-database! (mt/db))
- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")
- filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")
- ;; Step 2: Create a card with a filter on filter_col
- mp (lib-be/application-database-metadata-provider (mt/id))
- table (lib.metadata/table mp table-id)
- filter-field (lib.metadata/field mp filter-field-id)
- query (-> (lib/query mp table)
- (lib/filter (lib/= filter-field "value1")))]
- (mt/with-temp [:model/Card {card-id :id} {:dataset_query query
- :database_id (mt/id)}]
- ;; Create the dependency manually
- (t2/insert! :model/Dependency {:from_entity_type :card
- :from_entity_id card-id
- :to_entity_type :table
- :to_entity_id table-id})
- ;; Step 3: Run analysis - should succeed with no errors
- (let [card (t2/select-one :model/Card card-id)]
- ;; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see
- ;; old values after the sync!
- (lib-be/with-metadata-provider-cache
- (deps.findings/upsert-analysis! card)))
- (testing "Initial analysis succeeds"
- (is (true?
- (t2/select-one-fn :result :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)))
- (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))
- ;; Backdate the analysis timestamp and table/field updated_at so
- ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)
- ;; after a new sync makes an edit.
- ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and
- ;; the field update from sync would have identical timestamps.
- (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]
- (t2/update! :model/AnalysisFinding
- {:analyzed_entity_type :card :analyzed_entity_id card-id}
- {:analyzed_at old-time})
- (t2/update! :model/Field :table_id table-id
- {:updated_at old-time})
- (t2/update! :model/Table :id table-id
- {:updated_at old-time}))
- ;; Setup complete: Now call the thunk for the actual test run.
- (thunk {:db-id (:id (mt/db))
- :card-id card-id
- :table-id table-id
- :filter-field-id filter-field-id}))))))
-
-;; Integration test that a real DB sync triggers re-analysis of an updated table.
-(deftest ^:synchronized sync-removed-column-triggers-reanalysis-test
- (testing "When sync detects a removed column, re-analyzing the card shows errors"
- (with-syncable-db!
- (fn [{:keys [card-id filter-field-id]}]
- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- ;; Step 4: Remove the column used in the filter
- (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
- ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis
- ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.
- (sync/sync-database! (mt/db))
- ;; Verify the field is now inactive
- (testing "Field is marked inactive after sync"
- (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))
- ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.
- ;; Due to race conditions, the entity may be either stale (job hasn't run yet)
- ;; or already reanalyzed (job ran). Check for either valid outcome.
- (let [finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- (is (some? finding) "Analysis finding should exist")
- (if (:stale finding)
- ;; Job hasn't run yet - sync marked it stale, that's the expected behavior
- (testing "Entity is marked stale for re-analysis"
- (is (true? (:stale finding))))
- ;; Job already ran - verify the reanalysis detected the missing column
- (testing "Re-analysis detects the missing column"
- (is (false? (:result finding)) "Analysis should fail")
- (is (not= initial-analysis-time (:analyzed_at finding))
- "Analysis should have been re-run")
- (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]
- (is (seq errors) "Should have analysis errors")
- (is (some #(= :missing-column (:error_type %)) errors)
- "Should have a missing-column error"))))))))))
-
-;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all
-;; cards which depend on that table.
-(deftest ^:synchronized sync-without-changes-does-not-trigger-reanalysis-test
- (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"
- (with-syncable-db!
- (fn [{:keys [card-id db-id filter-field-id table-id]}]
- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)
- db-deps-checked (atom [])
- original-deps-check (mt/original-fn #'deps.events/synced-db->direct-dependents-of-changed-tables)
- table-before (into {} (t2/select-one :model/Table :id table-id))
- filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]
- ;; Re-sync the database, having made no changes to it.
- (mt/with-dynamic-fn-redefs [deps.events/synced-db->direct-dependents-of-changed-tables
- (fn [db-id]
- (u/prog1 (original-deps-check db-id)
- (swap! db-deps-checked conj [db-id <>])))]
- (sync/sync-database! (mt/db)))
- (testing "sync doesn't update tables or fields that haven't changed"
- (let [table-after (into {} (t2/select-one :model/Table :id table-id))
- filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]
- (is (= table-before table-after))
- (is (= filter-field-before filter-field-after))))
- (testing "the DB is were checked for tables that need deps updates, but none were found"
- (is (=? [[db-id empty?]]
- @db-deps-checked)))
- (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"
- (let [finding (t2/select-one :model/AnalysisFinding
- :analyzed_entity_type :card
- :analyzed_entity_id card-id)]
- (is (some? finding) "Analysis finding should exist")
- (is (true? (:result finding)) "Analysis should be a success")
- (is (= initial-analysis-time (:analyzed_at finding))
- "Analysis should not have been re-run"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
index dcefe1b4346..6c2e4ab094f 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj
@@ -185,7 +185,7 @@
(upstream-of :card (:id question-card))))))))))))
(deftest dependency-status-marked-stale-test
- (testing "dependency_status is marked stale when entities are created or updated"
+ (testing "dependency_status is marked stale when entities are updated, and not touched when they are created"
(mt/with-model-cleanup [:model/DependencyStatus]
(testing "cards"
(mt/with-model-cleanup [:model/Dependency]
@@ -197,7 +197,7 @@
(testing "transforms create"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/Transform transform]
- (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform)))))))
(testing "transforms update"
(mt/with-premium-features #{}
(mt/with-temp [:model/Transform transform]
@@ -207,7 +207,7 @@
(testing "snippets create"
(mt/with-premium-features #{:dependencies}
(mt/with-temp [:model/NativeQuerySnippet snippet]
- (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true)))))
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet)))))))
(testing "snippets update"
(mt/with-premium-features #{}
(mt/with-temp [:model/NativeQuerySnippet snippet]
@@ -215,13 +215,15 @@
(t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))
(is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true))))))))
-(deftest dependency-status-marked-stale-on-card-create-test
- (testing "dependency_status is marked stale when a card is created"
+(deftest dependency-status-absent-until-backfill-on-card-create-test
+ (testing "a created card has no dependency_status row until the backfill job processes it"
(mt/with-premium-features #{:dependencies}
(mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus]
(mt/with-temp [:model/User user]
(let [card (card/create-card! (basic-orders) user)]
- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true))))))))
+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card))))
+ (deps.test/synchronously-run-backfill!)
+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale false))))))))
(deftest filtered-graph-dependencies-test
(testing "filtered-graph-dependencies respects filter clause"
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj
new file mode 100644
index 00000000000..956d3f0e5bc
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj
@@ -0,0 +1,158 @@
+(ns metabase-enterprise.dependencies.sync-test
+ (:require
+ [clojure.java.jdbc :as jdbc]
+ [clojure.test :refer [deftest is testing]]
+ [java-time.api :as t]
+ [metabase-enterprise.dependencies.db :as deps.db]
+ [metabase-enterprise.dependencies.findings :as deps.findings]
+ [metabase-enterprise.dependencies.hooks]
+ [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]
+ [metabase.lib-be.core :as lib-be]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.sync.core :as sync]
+ [metabase.test :as mt]
+ [metabase.test.data.one-off-dbs :as one-off-dbs]
+ [metabase.util :as u]
+ [toucan2.core :as t2]))
+
+(defn- with-syncable-db! [thunk]
+ (mt/with-premium-features #{:dependencies}
+ (one-off-dbs/with-blank-db
+ ;; Step 1: Create a table with two columns
+ (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"
+ "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]
+ (jdbc/execute! one-off-dbs/*conn* [statement]))
+ ;; Sync the database to pick up the new table
+ (sync/sync-database! (mt/db))
+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")
+ filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")
+ ;; Step 2: Create a card with a filter on filter_col
+ mp (lib-be/application-database-metadata-provider (mt/id))
+ table (lib.metadata/table mp table-id)
+ filter-field (lib.metadata/field mp filter-field-id)
+ query (-> (lib/query mp table)
+ (lib/filter (lib/= filter-field "value1")))]
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query
+ :database_id (mt/id)}]
+ ;; Create the dependency manually
+ (t2/insert! :model/Dependency {:from_entity_type :card
+ :from_entity_id card-id
+ :to_entity_type :table
+ :to_entity_id table-id})
+ ;; Step 3: Run analysis - should succeed with no errors
+ (let [card (t2/select-one :model/Card card-id)]
+ ;; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see
+ ;; old values after the sync!
+ (lib-be/with-metadata-provider-cache
+ (deps.findings/upsert-analysis! card)))
+ (testing "Initial analysis succeeds"
+ (is (true?
+ (t2/select-one-fn :result :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)))
+ (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))
+ ;; Backdate the analysis timestamp and table/field updated_at so
+ ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)
+ ;; after a new sync makes an edit.
+ ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and
+ ;; the field update from sync would have identical timestamps.
+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]
+ (t2/update! :model/AnalysisFinding
+ {:analyzed_entity_type :card :analyzed_entity_id card-id}
+ {:analyzed_at old-time})
+ (t2/update! :model/Field :table_id table-id
+ {:updated_at old-time})
+ (t2/update! :model/Table :id table-id
+ {:updated_at old-time}))
+ ;; Setup complete: Now call the thunk for the actual test run.
+ (thunk {:db-id (:id (mt/db))
+ :card-id card-id
+ :table-id table-id
+ :filter-field-id filter-field-id}))))))
+
+;; Integration test that a real DB sync triggers re-analysis of an updated table.
+(deftest ^:synchronized sync-removed-column-triggers-reanalysis-test
+ (testing "When sync detects a removed column, re-analyzing the card shows errors"
+ (with-syncable-db!
+ (fn [{:keys [card-id filter-field-id]}]
+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ ;; Step 4: Remove the column used in the filter
+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
+ ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis
+ ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.
+ (sync/sync-database! (mt/db))
+ ;; Verify the field is now inactive
+ (testing "Field is marked inactive after sync"
+ (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))
+ ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.
+ ;; Due to race conditions, the entity may be either stale (job hasn't run yet)
+ ;; or already reanalyzed (job ran). Check for either valid outcome.
+ (let [finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ (is (some? finding) "Analysis finding should exist")
+ (if (:stale finding)
+ ;; Job hasn't run yet - sync marked it stale, that's the expected behavior
+ (testing "Entity is marked stale for re-analysis"
+ (is (true? (:stale finding))))
+ ;; Job already ran - verify the reanalysis detected the missing column
+ (testing "Re-analysis detects the missing column"
+ (is (false? (:result finding)) "Analysis should fail")
+ (is (not= initial-analysis-time (:analyzed_at finding))
+ "Analysis should have been re-run")
+ (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]
+ (is (seq errors) "Should have analysis errors")
+ (is (some #(= :missing-column (:error_type %)) errors)
+ "Should have a missing-column error"))))))))))
+
+(deftest ^:synchronized scheduled-metadata-sync-triggers-reanalysis-test
+ (testing "The metadata sync the scheduled sync task runs marks the dependents of a changed table for re-analysis"
+ (with-syncable-db!
+ (fn [{:keys [card-id]}]
+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])
+ (sync/sync-db-metadata! (mt/db))
+ (let [finding (t2/select-one :model/AnalysisFinding :analyzed_entity_type :card :analyzed_entity_id card-id)]
+ (is (or (:stale finding)
+ (not= initial-analysis-time (:analyzed_at finding))))))))))
+
+;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all
+;; cards which depend on that table.
+(deftest ^:synchronized sync-without-changes-does-not-trigger-reanalysis-test
+ (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"
+ (with-syncable-db!
+ (fn [{:keys [card-id db-id filter-field-id table-id]}]
+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)
+ db-deps-checked (atom [])
+ original-deps-check (mt/original-fn #'deps.db/table-ids-with-outdated-findings)
+ table-before (into {} (t2/select-one :model/Table :id table-id))
+ filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]
+ ;; Re-sync the database, having made no changes to it.
+ (mt/with-dynamic-fn-redefs [deps.db/table-ids-with-outdated-findings
+ (fn [db-id]
+ (u/prog1 (original-deps-check db-id)
+ (swap! db-deps-checked conj [db-id <>])))]
+ (sync/sync-database! (mt/db)))
+ (testing "sync doesn't update tables or fields that haven't changed"
+ (let [table-after (into {} (t2/select-one :model/Table :id table-id))
+ filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]
+ (is (= table-before table-after))
+ (is (= filter-field-before filter-field-after))))
+ (testing "the DB is were checked for tables that need deps updates, but none were found"
+ (is (=? [[db-id empty?]]
+ @db-deps-checked)))
+ (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"
+ (let [finding (t2/select-one :model/AnalysisFinding
+ :analyzed_entity_type :card
+ :analyzed_entity_id card-id)]
+ (is (some? finding) "Analysis finding should exist")
+ (is (true? (:result finding)) "Analysis should be a success")
+ (is (= initial-analysis-time (:analyzed_at finding))
+ "Analysis should not have been re-run"))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
index 58deb44af0f..e84d4e5159f 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
@@ -1,6 +1,7 @@
(ns metabase-enterprise.dependencies.task.backfill-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.dependencies.task.backfill-test]}}}}}}
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[environ.core :as env]
[java-time.api :as t]
@@ -9,7 +10,7 @@
[metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
[metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
[metabase-enterprise.dependencies.test-util :as deps.test]
- [metabase.events.core :as events]
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
[metabase.test :as mt]
@@ -34,7 +35,7 @@
(defn- mark-stale!
"Mark an entity as stale in dependency_status for testing."
[entity-type entity-id]
- (deps.dependency-status/mark-stale! entity-type [entity-id]))
+ (deps.dependency-status/mark-stale! {entity-type #{entity-id}}))
(defn- assert-processed
"Assert that an entity has been processed (not stale, current version)."
@@ -374,33 +375,6 @@
(assert-processed :card card-id)
(is (= 1 deps-after))))))))
-(deftest ^:synchronized backfill-dependencies-on-serdes-load-test
- (testing "Serdes load triggers the backfill job (entities remain stale until job runs)"
- (backfill-all-existing-entities!)
- (let [query (mt/mbql-query orders)]
- (mt/with-premium-features #{:dependencies}
- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}
- :model/Card {card2-id :id} {:dataset_query query}
- :model/Card {card3-id :id} {:dataset_query query}]
- ;; Mark all as stale
- (mark-stale! :card card1-id)
- (mark-stale! :card card2-id)
- (mark-stale! :card card3-id)
- ;; serdes-load triggers the async job, not synchronous processing
- (events/publish-event! :event/serdes-load {})
- ;; entities are still stale (job hasn't run yet in this test)
- (is (= 3 (t2/count :model/DependencyStatus
- :entity_type :card
- :entity_id [:in [card1-id card2-id card3-id]]
- :stale true)))
- ;; manually run the backfill to verify it processes them
- (backfill-all-existing-entities!)
- (is (= 3 (t2/count :model/DependencyStatus
- :entity_type :card
- :entity_id [:in [card1-id card2-id card3-id]]
- :stale false
- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))
-
(deftest ^:synchronized backfill-dependencies-on-token-update-test
(testing "Token update triggers the backfill job (entities remain stale until job runs)"
(backfill-all-existing-entities!)
@@ -414,7 +388,7 @@
(mark-stale! :card card2-id)
(mark-stale! :card card3-id)
;; token update triggers the async job, not synchronous processing
- (events/publish-event! :event/set-premium-embedding-token {})
+ (dependencies.hooks/after-premium-token-update)
;; entities are still stale (job hasn't run yet in this test)
(is (= 3 (t2/count :model/DependencyStatus
:entity_type :card
@@ -485,7 +459,10 @@
(reset! scheduled [])
(thunk)
(count @scheduled))]
- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args]
+ (when (some #(str/includes? (str %) "dependency-backfill") args)
+ (swap! scheduled conj args))
+ nil)]
(with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "0")]
(testing "the 1-second event-driven trigger is suppressed"
(is (zero? (scheduled-by dependencies.backfill/trigger-backfill-job!))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj
index a7c89c87e99..aa770887e24 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj
@@ -3,7 +3,7 @@
[clojure.test :refer [deftest is testing]]
[environ.core :as env]
[metabase-enterprise.dependencies.task.entity-check :as dependencies.entity-check]
- [metabase.events.core :as events]
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.premium-features.core :as premium-features]
[metabase.task.core :as task]
[metabase.test :as mt])
@@ -61,18 +61,15 @@
(def ^:private entity-check-job-name "metabase.dependencies.task.entity-check.job")
(defn- scheduled-job-names
- "Job names from captured [[task/schedule-task!]] calls. Needed because several jobs listen for the same events, so a
- bare call count cannot tell us which job got scheduled."
+ "Job names from captured [[task/schedule-task!]] calls, since a token update schedules both dependency jobs."
[calls]
(into #{} (map (fn [args] (.getName (.getKey ^JobDetail (nth args 1))))) calls))
-(deftest ^:synchronized token-event-restarts-the-job-test
- (testing "Setting a premium token must restart the entity check job. The backfill job derives
- :event/set-premium-embedding-token for this; entity check had no event wiring at all, so its only restarts
- came from content-change handlers. Enabling the feature on an idle instance left the periodic checker dead
- until the next process restart."
+(deftest ^:synchronized token-update-restarts-the-job-test
+ (testing "Setting a premium token must restart the entity check job, or enabling the feature on an idle instance
+ leaves the periodic checker dead until the next process restart."
(let [scheduled (atom [])]
(mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]
(mt/with-premium-features #{:dependencies}
- (events/publish-event! :event/set-premium-embedding-token {})
+ (dependencies.hooks/after-premium-token-update)
(is (contains? (scheduled-job-names @scheduled) entity-check-job-name)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
index b122e09b05c..a61b623fa59 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj
@@ -2,6 +2,7 @@
(:require
[medley.core :as m]
[metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]
+ [metabase.app-db.core :as mdb]
[metabase.lib.convert :as lib.convert]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -16,6 +17,13 @@
(mt/with-premium-features #{:dependencies}
(while (#'dependencies.backfill/backfill-dependencies!))))
+(defn do-with-immediate-hook-writes
+ "Run `thunk` with the work the dependency hooks defer until commit done at once, since a test's transaction never
+ commits."
+ [thunk]
+ (mt/with-dynamic-fn-redefs [mdb/do-after-commit (fn [f] (f))]
+ (thunk)))
+
(defn mock-card [metadata-provider {:keys [id query details]}]
(merge {:lib/type :metadata/card
:id id
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
index e13d5e4f5a4..f9fe5207c5a 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj
@@ -1,7 +1,7 @@
(ns ^:mb/driver-tests metabase-enterprise.replacement.api-test
(:require
[clojure.test :refer :all]
- [metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.hooks]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.execute :as replacement.execute]
[metabase-enterprise.replacement.models.replacement-run :as replacement-run]
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
index 75a891c0182..61704ca4d83 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj
@@ -3,7 +3,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.replacement.runner-test]}}}}}}
(:require
[clojure.test :refer [deftest is testing]]
- [metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.hooks]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.protocols :as replacement.protocols]
[metabase-enterprise.replacement.runner :as replacement.runner]
@@ -20,9 +20,6 @@
(set! *warn-on-reflection* true)
-(comment
- metabase-enterprise.dependencies.events/keep-me)
-
(deftest bulk-load-metadata-for-entities-test
(testing "bulk-load-metadata-for-entities! fetches all entity types in bulk"
(mt/with-temp [:model/Card {card-id-1 :id} {:name "Card 1"
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
index 8a4c0ea27ba..bdb98152c40 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj
@@ -2,7 +2,7 @@
(:require
[clojure.test :refer [deftest is testing]]
[medley.core :as m]
- [metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.hooks]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.field-refs :as replacement.field-refs]
[metabase-enterprise.replacement.source-swap :as replacement.source-swap]
@@ -15,9 +15,6 @@
(set! *warn-on-reflection* true)
-(comment
- metabase-enterprise.dependencies.events/keep-me)
-
;;; ------------------------------------------------ Card --------------------------------------------------------
(deftest card-swap-source!-source-table-test
diff --git a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
index 59f9ad7d421..487366aa058 100644
--- a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj
@@ -1,7 +1,7 @@
(ns metabase-enterprise.replacement.usages-test
(:require
[clojure.test :refer [deftest is testing]]
- [metabase-enterprise.dependencies.events]
+ [metabase-enterprise.dependencies.hooks]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase-enterprise.replacement.usages :as replacement.usages]
[metabase.graph.core :as graph]
@@ -12,9 +12,6 @@
(set! *warn-on-reflection* true)
-(comment
- metabase-enterprise.dependencies.events/keep-me)
-
;;; ------------------------------------------------ In-memory graph helpers ------------------------------------------------
(defn- deps->adjacency-map
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
index 5d8f905fb15..16ca6181f34 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj
@@ -2,6 +2,7 @@
(:require
[clojure.java.io :as io]
[clojure.test :refer :all]
+ [metabase-enterprise.serialization.v2.ingest :as serdes.ingest]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.core :as mdb]
[metabase.app-db.schema-migrations-test.impl :as schema-migrations-test.impl]
@@ -545,3 +546,12 @@
:else (t2/select-fn-vec pk [model pk] {:where where}))]
(u/rfirst (serdes/extract-all model-name (cond-> {}
ids (assoc :filter-column pk :filter-ids ids))))))
+
+(defn ingestion-in-memory
+ "An Ingestable serving the extracted `entities`, in order, under their label-free paths as the YAML ingestion does."
+ [entities]
+ (let [by-path (u/index-by (comp serdes.ingest/strip-labels serdes/path) entities)]
+ (reify serdes.ingest/Ingestable
+ (ingest-list [_] (map (comp serdes.ingest/strip-labels serdes/path) entities))
+ (ingest-one [_ path] (by-path (serdes.ingest/strip-labels path)))
+ (ingest-errors [_] []))))
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
index fbef6cfcfdd..aebc7ad35f3 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj
@@ -1694,7 +1694,7 @@
(map serdes/path)
(into #{})))))
(testing "the click behavior looks sane"
- (is (= #{{:column_settings nil}
+ (is (= #{{}
{:column_settings {"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {}
"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Field To Click 2\"],null]]" {}}}}
(->> {:targets [["Collection" coll-id]]
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
index 20171203b05..bcc717d7a4d 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj
@@ -34,9 +34,6 @@
(binding [models.database/*include-h2-in-extract?* true]
(thunk)))))
-(defn- no-labels [path]
- (mapv #(dissoc % :label) path))
-
(defn- by-model [entities model-name]
(filter #(-> % :serdes/meta last :model (= model-name))
entities))
@@ -46,19 +43,6 @@
(map (comp :id last :serdes/meta))
set))
-(defn- ingestion-in-memory [extractions]
- (let [mapped (into {} (for [entity (vec extractions)]
- [(no-labels (serdes/path entity))
- entity]))]
- (reify
- serdes.ingest/Ingestable
- (ingest-list [_]
- (map (comp no-labels serdes/path) extractions))
- (ingest-one [_ path]
- (get mapped (no-labels path)))
- (ingest-errors [_]
- []))))
-
;;; WARNING for test authors: [[extract/extract]] returns a lazy reducible value. To make sure you don't
;;; confound your tests with data from your dev appdb, remember to eagerly
;;; `(into [] (extract/extract ...))` in these tests.
@@ -104,7 +88,7 @@
(let [{:keys [db coll card]} @extracted
bad-card (assoc-in card [:dataset_query :stages 0 :template-tags]
{"id" {:type :tag-type-from-the-future :name "id" :display-name "ID" :id "abc-123"}})
- ingestion #(ingestion-in-memory [db coll bad-card])
+ ingestion #(ts/ingestion-in-memory [db coll bad-card])
ours? #(some (partial re-find #"does not match this Metabase's query schema") %)]
(testing "by default the schema check is what refuses the import"
(mt/with-empty-h2-app-db!
@@ -140,7 +124,7 @@
(ts/create! :model/DashboardCard :dashboard_id (:id dashboard) :card_id (:id model))
(reset! serialized (into [] (serdes.extract/extract {})))))
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory
+ (serdes.load/load-metabase! (ts/ingestion-in-memory
(for [entity @serialized]
(cond-> entity
(= "Action" (-> entity :serdes/meta last :model))
@@ -165,7 +149,7 @@
@serialized))))
(testing "loading into an empty database succeeds"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; the trash is serialized and loaded, so restrict to nil type
(let [colls (t2/select :model/Collection :type nil)]
(is (= 1 (count colls)))
@@ -173,7 +157,7 @@
(is (= eid1 (:entity_id (first colls)))))))
(testing "loading again into the same database does not duplicate"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [colls (t2/select :model/Collection :type nil)]
(is (= 1 (count colls)))
(is (= "Basic Collection" (:name (first colls))))
@@ -213,7 +197,7 @@
age (ts/create! :model/Field :name "age" :table_id (:id table))
email (ts/create! :model/Field :name "email" :table_id (:id table))]
(ts/create! :model/Dimension :field_id (:id email) :name "Local" :type :internal)
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (= "Renamed" (t2/select-one-fn :display_name :model/TableUserSettings :table_id (:id table))))
(is (= "edited" (t2/select-one-fn :description :model/FieldUserSettings :field_id (:id age))))
(testing "the imported Dimension replaces the Field's local one"
@@ -239,7 +223,7 @@
(into (extract "FieldUserSettings" :field_id [(:id age)]))
(into (extract "Dimension" :field_id [(:id email)]))))))
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [stub (t2/select-one :model/Database :name "source-only-db")
table (t2/select-one :model/Table :db_id (:id stub) :name "customers")]
(is (true? (:is_stub stub)))
@@ -264,7 +248,7 @@
(t2/insert! :model/FieldUserSettings {:field_id name-id :semantic_type :type/City})
(extract [price-id name-id]))]
(try
- (serdes.load/load-metabase! (ingestion-in-memory serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory serialized))
(let [stub (t2/select-one :model/Database :name db-name)
table-id (t2/select-one-pk :model/Table :db_id (:id stub) :schema "PUBLIC" :name "VENUES")
field-ids (t2/select-pks-vec :model/Field :table_id table-id {:order-by [[:name :asc]]})
@@ -308,7 +292,7 @@
(serdes/extract-one "Card" {} good)])))
(ts/with-db dest-db
(serdes/with-cache
- (serdes.load/load-metabase! (ingestion-in-memory (update @serialized 1 assoc :name nil))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory (update @serialized 1 assoc :name nil))
:continue-on-error true))
(let [stub (t2/select-one :model/Database :name "source-only-db")]
(is (true? (:is_stub stub)))
@@ -326,7 +310,7 @@
:name "Email" :type :internal)))
:serdes/meta)
field (ts/extract-one "Field" (:id email))
- load! (fn [entity] (serdes.load/load-metabase! (ingestion-in-memory [entity])))]
+ load! (fn [entity] (serdes.load/load-metabase! (ts/ingestion-in-memory [entity])))]
(t2/delete! :model/Dimension :field_id (:id email))
(load! (assoc field :dimensions [dimension]))
(is (= "Email" (t2/select-one-fn :name :model/Dimension :field_id (:id email))))
@@ -348,7 +332,7 @@
:fields [(ts/extract-one "FieldUserSettings" (:id age))]}]
(t2/delete! :model/FieldUserSettings :field_id (:id age))
(t2/insert! :model/FieldUserSettings {:field_id (:id email) :description "stale"})
- (serdes.load/load-metabase! (ingestion-in-memory [legacy]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [legacy]))
(is (= "edited" (t2/select-one-fn :description :model/FieldUserSettings :field_id (:id age))))
(is (not (t2/exists? :model/FieldUserSettings :field_id (:id email))))))))
@@ -361,7 +345,7 @@
_ (t2/insert! :model/FieldUserSettings {:field_id (:id email) :description "edited"})
settings (ts/extract-one "FieldUserSettings" (:id email))]
(t2/update! :model/FieldUserSettings (:id email) {:display_name "Local"})
- (serdes.load/load-metabase! (ingestion-in-memory [settings]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [settings]))
(is (=? {:description "edited" :display_name nil}
(t2/select-one :model/FieldUserSettings :field_id (:id email))))))))
@@ -376,7 +360,7 @@
extracted (ts/extract-one "Dimension" (:entity_id dimension))
tables (t2/count :model/Table)]
(t2/delete! :model/Dimension :field_id (:id tier))
- (serdes.load/load-metabase! (ingestion-in-memory [extracted]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [extracted]))
(is (= [(:id tier)] (t2/select-fn-vec :field_id :model/Dimension :entity_id (:entity_id dimension))))
(is (= tables (t2/count :model/Table)))))))
@@ -423,7 +407,7 @@
;; rather than aborting, and everything else lands. We do NOT promise a strict (default) import works.
(testing "import under continue-on-error doesn't crash; the dangling dashboard is skipped, the rest lands"
(ts/with-db dest-db
- (let [report (serdes.load/load-metabase! (ingestion-in-memory @serialized) {:continue-on-error true})]
+ (let [report (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized) {:continue-on-error true})]
(is (some? (t2/select-one :model/Collection :entity_id @coll-eid)) "target collection imported")
(is (some? (t2/select-one :model/Card :entity_id @clean-eid)) "clean card imported")
(is (nil? (t2/select-one :model/Card :entity_id @escaped-eid)) "escaped card not imported")
@@ -452,7 +436,7 @@
(ts/with-db dest-db
(ts/create! :model/Collection :name "Unrelated Collection")
(ts/create! :model/Collection :name "Parent Collection" :location "/" :entity_id (:entity_id @parent))
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [parent-dest (t2/select-one :model/Collection :entity_id (:entity_id @parent))
child-dest (t2/select-one :model/Collection :entity_id (:entity_id @child))
grandchild-dest (t2/select-one :model/Collection :entity_id (:entity_id @grandchild))]
@@ -514,7 +498,7 @@
:parent_id))))
(testing "deserialization works properly, keeping the same-named tables apart"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(reset! db1d (t2/select-one :model/Database :name (:name @db1s)))
(reset! db2d (t2/select-one :model/Database :name (:name @db2s)))
(is (= 3 (t2/count :model/Database)))
@@ -589,7 +573,7 @@
(reset! field2d (ts/create! :model/Field :name "subtotal" :table_id (:id @table2d)))
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"))
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(reset! db1d (t2/select-one :model/Database :name "my-db"))
(reset! table1d (t2/select-one :model/Table :name "customers"))
@@ -652,7 +636,7 @@
field (ts/create! :model/Field :name "category" :table_id (:id table)
:base_type :type/Text :semantic_type :type/Category
:has_field_values :list)]
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [dest-card (t2/select-one :model/Card :name "Products Model")
cat (first (filter #(= "category" (:name %)) (:result_metadata dest-card)))]
(is (number? (:id cat))
@@ -694,7 +678,7 @@
(testing "deserializing synthesizes inactive Table and Field for the dangling references"
(ts/with-db dest-db
(ts/create! :model/Database :name "my-db")
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [db (t2/select-one :model/Database :name "my-db")
table (t2/select-one :model/Table :name "customers" :db_id (:id db))
field (and table (t2/select-one :model/Field :name "age" :table_id (:id table)))
@@ -756,7 +740,7 @@
(reset! field2d (ts/create! :model/Field :name "subtotal" :table_id (:id @table2d)))
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"))
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(reset! db1d (t2/select-one :model/Database :name "my-db"))
(reset! table1d (t2/select-one :model/Table :name "customers"))
@@ -840,7 +824,7 @@
_ (ts/create! :model/Field :name "subtotal" :table_id (:id table2d))
_ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")]
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(let [db1d (t2/select-one :model/Database :name "my-db")
table1d (t2/select-one :model/Table :name "sales")
@@ -904,7 +888,7 @@
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
;; Load the serialized content
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(let [db1d (t2/select-one :model/Database :name "my-db")
table1d (t2/select-one :model/Table :name "sales")
@@ -967,7 +951,7 @@
(ts/with-db dest-db
(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"))
;; Load the serialized content
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(reset! db1d (t2/select-one :model/Database :name "my-db"))
(reset! table1d (t2/select-one :model/Table :name "products"))
@@ -1182,7 +1166,7 @@
(ts/create! :model/Field :name "address" :table_id (:id @table2d))
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"))
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(reset! db1d (t2/select-one :model/Database :name "my-db"))
(reset! table1d (t2/select-one :model/Table :name "orders"))
@@ -1302,7 +1286,7 @@
:creator_id (:id @user1d)
:timezone "America/New_York"))
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(reset! timeline2d (t2/select-one :model/Timeline :entity_id (:entity_id @timeline2s)))
(reset! eventsT1 (t2/select :model/TimelineEvent :timeline_id (:id @timeline1d)))
@@ -1344,7 +1328,7 @@
:timestamp "2027-04-20T00:00:00Z"
:time_matters false
:timezone "UTC"}]}]
- (serdes.load/load-metabase! (ingestion-in-memory [timeline]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [timeline]))
(let [timeline-id (t2/select-one-pk :model/Timeline :entity_id timeline-eid)
[event] (t2/select :model/TimelineEvent :timeline_id timeline-id)]
(is (= 1 (t2/count :model/TimelineEvent :timeline_id timeline-id)))
@@ -1386,7 +1370,7 @@
(ts/create! :model/Dashboard :name "Other dashboard C")
(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"))
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(reset! dash1d (t2/select-one :model/Dashboard :name "My Dashboard"))
(reset! dash2d (t2/select-one :model/Dashboard :name "Linked dashboard"))
(testing "the Dashboards and Users have different IDs now"
@@ -1475,7 +1459,7 @@
;; The :values are different here; they should get overwritten by the update.
(reset! fv1d (ts/create! :model/FieldValues :field_id (:id @field1d) :values ["WA" "NC" "NM" "WI"]))
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(reset! fv1d (t2/select-one :model/FieldValues :field_id (:id @field1d)))
(reset! fv2d (t2/select-one :model/FieldValues :field_id (:id @field2d)))
@@ -1527,7 +1511,7 @@
(testing "deserializing restores collection_id correctly"
(ts/with-db dest-db
;; Load the serialized content
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Fetch the relevant bits
(reset! db1d (t2/select-one :model/Database :name "my-db"))
(reset! coll1d (t2/select-one :model/Collection :name "Publishing Collection"))
@@ -1550,7 +1534,7 @@
collection-eid (u/generate-nano-id)
card-path [{:model "Card" :id card-eid}]
timeline-path [{:model "Timeline" :id timeline-eid}]
- archive (ingestion-in-memory
+ archive (ts/ingestion-in-memory
[{:serdes/meta card-path
:entity_id card-eid
:name "Question with events"
@@ -1591,32 +1575,32 @@
(ts/create! :model/Field :name "STATE" :table_id (:id @table1s))
(ts/create! :model/User :first_name "Geddy" :last_name "Lee" :email "[email protected]")
(testing "depending on existing values works"
- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
- :created_at (t/instant)
- :creator_id "[email protected]"
- :database_id "my-db"
- :dataset_query {:database "my-db"
- :type :query
- :query {:source-table ["my-db" nil "CUSTOMERS"]}}
- :display :table
- :entity_id "0123456789abcdef_0123"
- :name "Some card"
- :table_id ["my-db" nil "CUSTOMERS"]
- :visualization_settings {}}])]
+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
+ :created_at (t/instant)
+ :creator_id "[email protected]"
+ :database_id "my-db"
+ :dataset_query {:database "my-db"
+ :type :query
+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}
+ :display :table
+ :entity_id "0123456789abcdef_0123"
+ :name "Some card"
+ :table_id ["my-db" nil "CUSTOMERS"]
+ :visualization_settings {}}])]
(is (some? (serdes.load/load-metabase! ingestion)))))
(testing "depending on a nonexisting database creates a stub database"
- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
- :created_at (t/instant)
- :creator_id "[email protected]"
- :database_id "bad-db"
- :dataset_query {:database "bad-db"
- :type :query
- :query {:source-table ["bad-db" nil "CUSTOMERS"]}}
- :display :table
- :entity_id "0123456789abcdef_0123"
- :name "Some card"
- :table_id ["bad-db" nil "CUSTOMERS"]
- :visualization_settings {}}])
+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]
+ :created_at (t/instant)
+ :creator_id "[email protected]"
+ :database_id "bad-db"
+ :dataset_query {:database "bad-db"
+ :type :query
+ :query {:source-table ["bad-db" nil "CUSTOMERS"]}}
+ :display :table
+ :entity_id "0123456789abcdef_0123"
+ :name "Some card"
+ :table_id ["bad-db" nil "CUSTOMERS"]
+ :visualization_settings {}}])
_ (serdes.load/load-metabase! ingestion)
stub (t2/select-one :model/Database :name "bad-db")]
(is (=? {:engine :postgres
@@ -1629,19 +1613,19 @@
(is (= (:id stub)
(t2/select-one-fn :database_id :model/Card :entity_id "0123456789abcdef_0123")))))
(testing "depending on nonexisting values fails"
- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0124"}]
- :collection_id "missingcollection0001"
- :created_at (t/instant)
- :creator_id "[email protected]"
- :database_id "my-db"
- :dataset_query {:database "my-db"
- :type :query
- :query {:source-table ["my-db" nil "CUSTOMERS"]}}
- :display :table
- :entity_id "0123456789abcdef_0124"
- :name "Other card"
- :table_id ["my-db" nil "CUSTOMERS"]
- :visualization_settings {}}])
+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0124"}]
+ :collection_id "missingcollection0001"
+ :created_at (t/instant)
+ :creator_id "[email protected]"
+ :database_id "my-db"
+ :dataset_query {:database "my-db"
+ :type :query
+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}
+ :display :table
+ :entity_id "0123456789abcdef_0124"
+ :name "Other card"
+ :table_id ["my-db" nil "CUSTOMERS"]
+ :visualization_settings {}}])
e (try
(serdes.load/load-metabase! ingestion)
nil
@@ -1658,21 +1642,21 @@
(testing "attaching the referrer does not duplicate the error through the cause chain"
(is (nil? (ex-cause e))))))
(testing "a result_metadata table_id naming an absent database creates a stub database"
- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0125"}]
- :created_at (t/instant)
- :creator_id "[email protected]"
- :database_id "my-db"
- :dataset_query {:database "my-db"
- :type :query
- :query {:source-table ["my-db" nil "CUSTOMERS"]}}
- :display :table
- :entity_id "0123456789abcdef_0125"
- :name "Some card"
- :result_metadata [{:name "STATE"
- :base_type :type/Text
- :table_id ["absent-db" nil "CUSTOMERS"]}]
- :table_id ["my-db" nil "CUSTOMERS"]
- :visualization_settings {}}])]
+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0125"}]
+ :created_at (t/instant)
+ :creator_id "[email protected]"
+ :database_id "my-db"
+ :dataset_query {:database "my-db"
+ :type :query
+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}
+ :display :table
+ :entity_id "0123456789abcdef_0125"
+ :name "Some card"
+ :result_metadata [{:name "STATE"
+ :base_type :type/Text
+ :table_id ["absent-db" nil "CUSTOMERS"]}]
+ :table_id ["my-db" nil "CUSTOMERS"]
+ :visualization_settings {}}])]
(serdes.load/load-metabase! ingestion)
(is (true? (t2/select-one-fn :is_stub :model/Database :name "absent-db")))))))))
@@ -1707,7 +1691,7 @@
(:dataset_query @extracted))))
(testing "when loading"
(let [new-eid (u/generate-nano-id)
- ingestion (ingestion-in-memory [(assoc @extracted :entity_id new-eid)])]
+ ingestion (ts/ingestion-in-memory [(assoc @extracted :entity_id new-eid)])]
(is (some? (serdes.load/load-metabase! ingestion)))
(is (= (:id @snippet1s)
(-> (t2/select-one :model/Card :entity_id new-eid)
@@ -1724,7 +1708,7 @@
id1 (u/generate-nano-id)
id2 (u/generate-nano-id)
load! #(serdes.load/load-metabase!
- (ingestion-in-memory [(serdes/extract-one "NativeQuerySnippet" {} %)]))]
+ (ts/ingestion-in-memory [(serdes/extract-one "NativeQuerySnippet" {} %)]))]
(testing "setup is correct"
(is (= (:entity_id snippet)
(t2/select-one-fn :entity_id :model/NativeQuerySnippet :name unique-name))))
@@ -1759,7 +1743,7 @@
(:template_tags (first @serialized))))))
(ts/with-db dest-db
(ts/create! :model/NativeQuerySnippet :name "Unrelated" :content "3 = 3")
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (=? {"snippet: A" {:snippet-id (t2/select-one-pk :model/NativeQuerySnippet :name "A")}}
(t2/select-one-fn :template_tags :model/NativeQuerySnippet :name "B"))))))))
@@ -1774,7 +1758,7 @@
:content "WHERE id = {{id}}"
:creator_id "[email protected]"
:entity_id "test-entity-1"}
- ingestion (ingestion-in-memory [snippet-data])]
+ ingestion (ts/ingestion-in-memory [snippet-data])]
(serdes.load/load-metabase! ingestion)
(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-1")]
;; Toucan hooks compute the template-tags:
@@ -1792,7 +1776,7 @@
:template_tags {}
:creator_id "[email protected]"
:entity_id "test-entity-2"}
- ingestion (ingestion-in-memory [snippet-data])]
+ ingestion (ts/ingestion-in-memory [snippet-data])]
(serdes.load/load-metabase! ingestion)
(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-2")]
(is (= {} template-tags))))))
@@ -1810,7 +1794,7 @@
:display-name "Snippet: WOOP"}}
:creator_id "[email protected]"
:entity_id "test-entity-3"}
- ingestion (ingestion-in-memory [snippet-data])]
+ ingestion (ts/ingestion-in-memory [snippet-data])]
(serdes.load/load-metabase! ingestion)
(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-3")]
(is (=? {"snippet: id" {:type :snippet
@@ -1874,7 +1858,7 @@
:dataset_query {:type :query
:database (:id db)
:query {:source-table (:id table)}})))
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [target-id (t2/select-one-fn :id :model/Card :entity_id @target-eid)
new-tag-name (str "#" target-id "-bh-population-model")
stored-query #(t2/select-one-fn :dataset_query :model/Card :entity_id @native-eid)
@@ -1890,7 +1874,7 @@
:card-id target-id}]
(-> query :stages first :template-tags)))
(testing "re-importing over the existing card (the update path) repairs the same way"
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (= query (stored-query))))))))))
(deftest load-action-test
@@ -1924,7 +1908,7 @@
(is (string? (:type action-serialized))))))))
(testing "loading succeeds"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [action (action/select-action :entity_id eid)]
(is (some? action))
(is (some? (:dataset_query action)))
@@ -1953,7 +1937,7 @@
(reset! dashcard2d (ts/create! :model/DashboardCard :dashboard_id (:id @dash1d)))
(reset! tab2d (ts/create! :model/DashboardTab :name "Tab 2" :dashboard_id (:id @dash1d)))
;; Load the serialized content.
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(reset! dash1d (-> (t2/select-one :model/Dashboard :name "My Dashboard")
(t2/hydrate :dashcards)
(t2/hydrate :tabs)))
@@ -1988,7 +1972,7 @@
series3s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card3s) :position 2)
extract1 (into [] (serdes.extract/extract {:no-settings true}))]
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory extract1))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory extract1))
(ts/with-db source-db
;; delete the 1st series and update the 3rd series to have position 0, and the 2nd series to have position 1
(t2/delete! :model/DashboardCardSeries (:id series1s))
@@ -2003,7 +1987,7 @@
(testing "Sense check: there are 3 series for the dashboard card initially"
(is (= 3
(t2/count :model/DashboardCardSeries :dashboardcard_id (:dashboardcard_id series-to-be-deleted)))))
- (serdes.load/load-metabase! (ingestion-in-memory extract2))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory extract2))
(let [dash1d (-> (t2/select-one :model/Dashboard :name "My Dashboard")
(t2/hydrate [:dashcards :series]))]
(testing "Dashboard cards have the same entity ID"
@@ -2035,7 +2019,7 @@
:model/DashboardCardSeries _s2 {:dashboardcard_id (:id dc2) :card_id (:id sc) :position 0}]
(let [extract (into [] (serdes.extract/extract {:no-settings true}))]
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory extract))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory extract))
(testing "Both series get imported even though they point at the same card"
(is (= 2
(t2/count :model/DashboardCardSeries)))))))))))
@@ -2072,7 +2056,7 @@
(into []))))))
(testing "The extraneous keys do not interfere with loading"
(ts/with-db dest-db
- (is (serdes.load/load-metabase! (ingestion-in-memory @serialized))))))))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))))))))
(deftest tx-test
(mt/with-empty-h2-app-db!
@@ -2085,7 +2069,7 @@
vec))
(testing "Load completes successfully"
(t2/update! :model/Card {:id (:id card)} {:name (str "qwe_" (:name card))})
- (is (serdes.load/load-metabase! (ingestion-in-memory @serialized)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized)))
(is (= (:name card)
(t2/select-one-fn :name :model/Card :id (:id card)))))
(testing "Partial load commits successful entities; failed entity does not persist"
@@ -2098,7 +2082,7 @@
(throw (ex-info "oops, error" {}))
(load-update! model adjusted local)))]
(is (thrown? clojure.lang.ExceptionInfo
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))))
;; Collection loaded successfully in its own transaction — committed despite Card failure
(is (= (:name coll)
(t2/select-one-fn :name :model/Collection :id (:id coll))))
@@ -2133,7 +2117,7 @@
"40001" ; SQL state
40001))))) ; H2 error code
(load-update! model adjusted local))]
- (is (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
"Import should succeed after retrying the deadlocked entity")
(is (= (:name card)
(t2/select-one-fn :name :model/Card :id (:id card)))
@@ -2149,7 +2133,7 @@
(throw (ex-info "constraint violation" {})))
(load-update! model adjusted local))]
(is (thrown? clojure.lang.ExceptionInfo
- (serdes.load/load-metabase! (ingestion-in-memory @serialized)))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized)))
"Non-transient error should propagate")
(is (= 1 @call-count)
"Should not retry non-transient errors"))))
@@ -2163,7 +2147,7 @@
(throw (ex-info "oops" {}))
(load-update! model adjusted local)))]
(is (thrown? clojure.lang.ExceptionInfo
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))))
;; With per-entity transactions, the Collection commit survives the Card failure.
;; On master (single transaction), the Collection update is rolled back.
(is (= (:name coll)
@@ -2217,7 +2201,7 @@
select-target #(-> % :visualization_settings :click_behavior :targetId)]
;; loaded to the same db so we know ids won't be corrupted
(testing "Dashcard ids won't be changed by loading on top of them"
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(is (= (:id dc1)
(t2/select-one-fn :id :model/DashboardCard :entity_id (:entity_id dc1))))
(is (= (:id dc2)
@@ -2233,7 +2217,7 @@
;; loaded to a different db so we know it works for sure instead of relying on data being present in the
;; database
(ts/with-db dest-db
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(testing "Circular dependencies work for Dashboard Questions as well"
(is (= (t2/select-one-fn :id :model/Dashboard :entity_id (:entity_id dash1))
(t2/select-one-fn :dashboard_id :model/Card :entity_id (:entity_id card-2))))))))))
@@ -2259,13 +2243,13 @@
ser (into [] (serdes.extract/extract {:no-settings true
:no-data-model false}))]
(testing "loading on top of the existing card"
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(is (= (:id card)
(-> (t2/select-one :model/Card :entity_id (:entity_id card))
:parameters first :values_source_config :card_id))))
(testing "loading into an empty database"
(ts/with-db dest-db
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(let [new-card (t2/select-one :model/Card :entity_id (:entity_id card))]
(is (= (:id new-card)
(-> new-card :parameters first :values_source_config :card_id)))))))))))
@@ -2282,7 +2266,7 @@
ser (into [] (serdes.extract/extract {:no-settings true
:no-data-model false}))]
(testing "loading on top of the existing cards"
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(is (= (:id card-b)
(-> (t2/select-one :model/Card :entity_id (:entity_id card-a))
:parameters first :values_source_config :card_id)))
@@ -2291,7 +2275,7 @@
:parameters first :values_source_config :card_id))))
(testing "loading into an empty database"
(ts/with-db dest-db
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(let [new-a (t2/select-one :model/Card :entity_id (:entity_id card-a))
new-b (t2/select-one :model/Card :entity_id (:entity_id card-b))]
(is (= (:id new-b)
@@ -2312,13 +2296,13 @@
ser (into [] (serdes.extract/extract {:no-settings true
:no-data-model false}))]
(testing "loading on top of the existing dashboard"
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(is (= (:id card)
(-> (t2/select-one :model/Dashboard :entity_id (:entity_id dash))
:parameters first :values_source_config :card_id))))
(testing "loading into an empty database"
(ts/with-db dest-db
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(let [new-dash (t2/select-one :model/Dashboard :entity_id (:entity_id dash))
new-card (t2/select-one :model/Card :entity_id (:entity_id card))]
(is (= (:id new-card)
@@ -2345,19 +2329,19 @@
messages (fn [e] (map ex-message (take-while some? (iterate ex-cause e))))]
(ts/with-db dest-db
(testing "A dashboard question loads into Library Dashboards before its dashboard"
- (is (serdes.load/load-metabase! (ingestion-in-memory (cons (card-ser dq) (remove #{(card-ser standalone)} ser)))))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory (cons (card-ser dq) (remove #{(card-ser standalone)} ser)))))
(is (= (t2/select-one-pk :model/Dashboard :entity_id (:entity_id dash))
(t2/select-one-fn :dashboard_id :model/Card :entity_id (:entity_id dq)))))
(testing "A standalone question can't load into Library Dashboards"
(let [e (is (thrown? clojure.lang.ExceptionInfo
(serdes.load/load-metabase!
- (ingestion-in-memory [(assoc (card-ser standalone) :collection_id (:entity_id coll))]))))]
+ (ts/ingestion-in-memory [(assoc (card-ser standalone) :collection_id (:entity_id coll))]))))]
(is (some #(re-find #"Can only add dashboards to the 'Dashboards' collection" %) (messages e)))
(is (not (t2/exists? :model/Card :entity_id (:entity_id standalone))))))
(testing "A dashboard question can't become a standalone question in Library Dashboards"
(let [e (is (thrown? clojure.lang.ExceptionInfo
(serdes.load/load-metabase!
- (ingestion-in-memory [(assoc (card-ser dq) :dashboard_id nil)]))))]
+ (ts/ingestion-in-memory [(assoc (card-ser dq) :dashboard_id nil)]))))]
(is (some #(re-find #"Can only add dashboards to the 'Dashboards' collection" %) (messages e)))))))))))
(deftest continue-on-error-test
@@ -2396,7 +2380,7 @@
:no-transforms true}))
changed (change-ser ser {(:entity_id c2) {:collection_id "does-not-exist"}})]
(mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]
- (let [report (serdes.load/load-metabase! (ingestion-in-memory changed) {:continue-on-error true})]
+ (let [report (serdes.load/load-metabase! (ts/ingestion-in-memory changed) {:continue-on-error true})]
(is (= 1 (count (:errors report))))
(is (= 3 (count (:seen report)))))
(let [log-msgs (logs-extract #"Skipping deserialization error: (.*)" (messages))]
@@ -2426,12 +2410,12 @@
(is (not-any? :details dbs))
(ts/with-db dest-db
(testing "loading still works even if there are no details"
- (serdes.load/load-metabase! (ingestion-in-memory extracted))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory extracted))
(is (= {}
(t2/select-one-fn :details :model/Database)))
(testing "If we did not export details - it won't override existing data"
(t2/update! :model/Database {:details {:other "secret"}})
- (serdes.load/load-metabase! (ingestion-in-memory extracted))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory extracted))
(is (= {:other "secret"}
(t2/select-one-fn :details :model/Database)))))))))
(mt/with-temp [:model/Database _ {:name "My Database"
@@ -2453,7 +2437,7 @@
(mt/with-temp [:model/Dimension d2 {:name "Absolutely Other Dimension"
:field_id (mt/id :venues :price)
:type "internal"}]
- (serdes.load/load-metabase! (ingestion-in-memory ser))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory ser))
(is (= (:entity_id d1)
(t2/select-one-fn :entity_id :model/Dimension :field_id (mt/id :venues :price))))
(is (= nil
@@ -2484,7 +2468,7 @@
{:model "Field" :id "field"}]}
(ts/extract-one "Field" (:id f3))))
(t2/update! :model/Field (:id f3) {:description "some new one"})
- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))
+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))
(is (= "desc"
(t2/select-one-fn :description :model/Field (:id f3)))))))
@@ -2513,7 +2497,7 @@
(testing "the unlabeled Field exports no data_sensitivity key"
(is (not (contains? (field-ser @serialized "CONTACT_ROW") :data_sensitivity))))))
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [db-id (t2/select-one-fn :id :model/Database :name "labeled-db")
table-id (t2/select-one-fn :id :model/Table :name "CONTACTS" :db_id db-id)
email-id (t2/select-one-fn :id :model/Field :name "CONTACT_EMAIL" :table_id table-id)
@@ -2532,12 +2516,12 @@
[coll-ser] (serdes.extract/extract {:targets [["Collection" (:id coll)]]})
new-coll (assoc coll-ser :entity_id nil)
coll-count (fn [] (t2/count :model/Collection :name "mycoll"))]
- (serdes.load/load-metabase! (ingestion-in-memory [new-coll]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [new-coll]))
(is (= 2 (coll-count)))
- (serdes.load/load-metabase! (ingestion-in-memory [new-coll]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [new-coll]))
(is (= 3 (coll-count)))
(testing "absent :entity_id also works"
- (serdes.load/load-metabase! (ingestion-in-memory [(dissoc coll-ser :entity_id)]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(dissoc coll-ser :entity_id)]))
(is (= 4 (coll-count)))))))
(deftest import-published-table-with-existing-database-test
@@ -2571,7 +2555,7 @@
;; Pre-create the database on target
(let [target-db (ts/create! :model/Database :name "shared-db")]
;; Load the serialized content
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
;; Verify the table was imported correctly
(let [imported-table (t2/select-one :model/Table :name "published_table")
imported-coll (t2/select-one :model/Collection :entity_id @coll-eid)]
@@ -2604,7 +2588,7 @@
:no-settings true}))))))
(testing "import creates a stub database"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [stub (t2/select-one :model/Database :name "source-only-db")]
(is (=? {:engine :postgres
:details {}
@@ -2625,7 +2609,7 @@
(is (not (contains? @serialized :is_stub)))))
(ts/with-db dest-db
(let [real (ts/create! :model/Database :name "Prod" :engine :postgres :details {:host "prod"})]
- (serdes.load/load-metabase! (ingestion-in-memory [(assoc @serialized :is_stub true)]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(assoc @serialized :is_stub true)]))
(is (=? {:details {:host "prod"}
:is_stub false}
(t2/select-one :model/Database :id (:id real))))))))))
@@ -2650,7 +2634,7 @@
@serialized)]
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [segment (t2/select-one :model/Segment :name "Minors")
table (t2/select-one :model/Table :name "customers")]
(is (some? segment))
@@ -2676,7 +2660,7 @@
@serialized)]
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [measure (t2/select-one :model/Measure :name "Total Sales")
table (t2/select-one :model/Table :name "sales")]
(is (some? measure))
@@ -2705,7 +2689,7 @@
@serialized)]
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [card (t2/select-one :model/Card :name "Example Card")
db (t2/select-one :model/Database :name "my-db")
table (t2/select-one :model/Table :name "customers")]
@@ -2744,7 +2728,7 @@
@serialized)]
(ts/with-db dest-db
(t2/delete! :model/TransformTag)
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [transform (t2/select-one :model/Transform :name "Test Transform")
db (t2/select-one :model/Database :name "my-db")]
(is (some? transform))
@@ -2784,7 +2768,7 @@
(let [db (ts/create! :model/Database :name "padding-db")
table (ts/create! :model/Table :name "padding" :db_id (:id db))]
(ts/create! :model/Field :name "padding_field" :table_id (:id table)))
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [transform (t2/select-one :model/Transform :name "Checkpoint Transform")
field (t2/select-one :model/Field :name "updated_at")]
(is (some? transform))
@@ -2823,7 +2807,7 @@
(get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])))))
(ts/with-db dest-db
(t2/delete! :model/TransformTag)
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (= 999999999
(get-in (t2/select-one :model/Transform :name "Stale Checkpoint Transform")
[:source :source-incremental-strategy :checkpoint-filter-field-id]))
@@ -2867,7 +2851,7 @@
(is (some? (get-in extracted [:source :query])))))
(ts/with-db dest-db
(t2/delete! :model/TransformTag)
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [transform (t2/select-one :model/Transform :name "Orphan Transform")]
(testing "transform round-trips and ends up as a tombstone in the destination instance"
(is (some? transform))
@@ -2904,7 +2888,7 @@
(reset! serialized (into [] (serdes.extract/extract {})))))
(ts/with-db dest-db
(t2/delete! :model/TransformTag)
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [transform (t2/select-one :model/Transform :name "Hello Transform")
table (t2/select-one :model/Table :name "hello_transforms_world")]
(is (some? transform))
@@ -2924,7 +2908,7 @@
entity))
@serialized)]
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")]
(is (some? dashboard)))))))))
@@ -2949,7 +2933,7 @@
entity)))
@serialized)]
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")
dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))]
(is (some? dashboard))
@@ -2986,7 +2970,7 @@
entity)))
@serialized)]
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")
dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))
series (t2/select :model/DashboardCardSeries :dashboardcard_id (:id (first dashcards)))]
@@ -3010,7 +2994,7 @@
@serialized))))
(testing "loading into an empty database succeeds"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [channels (t2/select :model/Channel :name "Test Email Channel")]
(is (= 1 (count channels)))
(is (= "Test Email Channel" (:name (first channels))))
@@ -3018,7 +3002,7 @@
(is (= "A test email channel" (:description (first channels)))))))
(testing "loading again does not duplicate"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (= 1 (t2/count :model/Channel :name "Test Email Channel")))))))))
(deftest metabot-roundtrip-test
@@ -3038,14 +3022,14 @@
@serialized))))
(testing "loading into an empty database succeeds"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [metabots (t2/select :model/Metabot :entity_id @metabot-eid)]
(is (= 1 (count metabots)))
(is (= "Test Bot" (:name (first metabots))))
(is (= "A test metabot" (:description (first metabots)))))))
(testing "loading again does not duplicate"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (= 1 (t2/count :model/Metabot :entity_id @metabot-eid)))))))))
(deftest channel-minimal-required-properties-test
@@ -3064,7 +3048,7 @@
entity))
@serialized)]
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [channel (t2/select-one :model/Channel :name "Minimal Channel")]
(is (some? channel))
(is (= :channel/email (:type channel))))))))))
@@ -3084,7 +3068,7 @@
entity))
@serialized)]
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory minimal))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))
(let [metabot (t2/select-one :model/Metabot :name "Minimal Bot")]
(is (some? metabot))
(is (= "Minimal Bot" (:name metabot))))))))))
@@ -3120,7 +3104,7 @@
(is (not (contains? card-ser :query_type)) "query_type should be stripped from export"))))
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [card (t2/select-one :model/Card :name "Round Trip Card")
db (t2/select-one :model/Database :name "my-db")
table (t2/select-one :model/Table :name "orders")]
@@ -3147,7 +3131,7 @@
(is (not (contains? seg-ser :table_id)) "table_id should be stripped from export"))))
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [segment (t2/select-one :model/Segment :name "Minors")
table (t2/select-one :model/Table :name "customers")]
(is (some? segment))
@@ -3170,7 +3154,7 @@
(is (not (contains? msr-ser :table_id)) "table_id should be stripped from export"))))
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [measure (t2/select-one :model/Measure :name "Total Sales")
table (t2/select-one :model/Table :name "sales")]
(is (some? measure))
@@ -3202,7 +3186,7 @@
(is (contains? card-ser :database_id) "database_id exported — not derivable from empty query"))))
(ts/with-db dest-db
(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [card (t2/select-one :model/Card :name "Empty Query Card")
db (t2/select-one :model/Database :name "my-db")]
(is (some? card))
@@ -3265,7 +3249,7 @@
@serialized))))
(testing "loading into destination preserves library structure"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [lib-dest (t2/select-one :model/Collection :entity_id (:entity_id @library))
data-dest (t2/select-one :model/Collection :entity_id (:entity_id @data-root))
metrics-dest (t2/select-one :model/Collection :entity_id (:entity_id @metrics-root))
@@ -3325,7 +3309,7 @@
:object [:like (format "/collection/%d/%%" (:id data-dest))])))]
(testing "custom permissions exist before import"
(is (seq perms-before)))
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(let [data-after (t2/select-one :model/Collection :entity_id eid-data)
perms-after (set (map #(select-keys % [:group_id :object])
(t2/select :model/Permissions
@@ -3347,12 +3331,12 @@
(is (contains? (ids-by-model @serialized "Glossary") @eid)))
(testing "importing into an empty app db reproduces the term, definition and entity_id"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (=? [{:term "ARR" :definition "Annual recurring revenue" :entity_id @eid}]
(t2/select :model/Glossary)))))
(testing "importing again updates in place rather than duplicating"
(ts/with-db dest-db
- (serdes.load/load-metabase! (ingestion-in-memory @serialized))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))
(is (= 1 (t2/count :model/Glossary))))))))
(deftest glossary-load-find-local-test
@@ -3379,19 +3363,19 @@
(testing "a term-keyed file exported before entity_id existed updates the same-term row and keeps its entity_id"
(mt/with-empty-h2-app-db!
(let [{local-eid :entity_id} (ts/create! :model/Glossary :term "ARR" :definition "local")]
- (serdes.load/load-metabase! (ingestion-in-memory [(glossary-file "ARR" nil)]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(glossary-file "ARR" nil)]))
(is (=? [{:term "ARR" :definition "Annual recurring revenue (imported)" :entity_id local-eid}]
(t2/select :model/Glossary))))))
(testing "a file whose term exists locally under another entity_id updates that row in place and adopts the file's entity_id"
(mt/with-empty-h2-app-db!
(let [file-eid "glossaryfileeid000001"
{local-id :id} (ts/create! :model/Glossary :term "ARR" :definition "local")]
- (serdes.load/load-metabase! (ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))
(is (=? [{:id local-id :term "ARR" :definition "Annual recurring revenue (imported)" :entity_id file-eid}]
(t2/select :model/Glossary))))))
(testing "a file whose entity_id and term are both new inserts a row"
(mt/with-empty-h2-app-db!
(let [file-eid "glossaryfileeid000002"]
- (serdes.load/load-metabase! (ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))
+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))
(is (=? [{:term "ARR" :entity_id file-eid}]
(t2/select :model/Glossary))))))))
diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.clj
index 290fb3a5278..cc9e83d4427 100644
--- a/mage/src/mage/modules.clj
+++ b/mage/src/mage/modules.clj
@@ -151,6 +151,7 @@
custom-viz-plugin
dashboards
data-apps
+ dependencies
documents
api.eid-translation
embedding
diff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.clj
index 0c514721e41..17a48fed6e4 100644
--- a/src/metabase/batch_processing/impl.clj
+++ b/src/metabase/batch_processing/impl.clj
@@ -109,6 +109,8 @@
[:id :int]]
sdk-tracked-row
id+timestamp
+ [:map {:closed true}
+ [:table-id ms/PositiveInt]]
[:map {:closed true}
[:host :string]
[:timestamp (ms/InstanceOfClass java.time.temporal.Temporal)]]
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index c999db3b42a..1f81ab70485 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -49,7 +49,8 @@
(derive :metabase/model)
(derive :perms/use-parent-collection-perms)
(derive :hook/timestamped?)
- (derive :hook/entity-id))
+ (derive :hook/entity-id)
+ (derive :hook/dependencies))
(defmethod mi/can-write? :model/Dashboard
([instance]
diff --git a/src/metabase/dashboards/models/dashboard_card.clj b/src/metabase/dashboards/models/dashboard_card.clj
index d4a38767129..582b0cab5c8 100644
--- a/src/metabase/dashboards/models/dashboard_card.clj
+++ b/src/metabase/dashboards/models/dashboard_card.clj
@@ -24,6 +24,7 @@
(derive ::mi/write-policy.full-perms-for-perms-set)
(derive :hook/timestamped?)
(derive :hook/entity-id)
+ (derive :hook/dependencies)
;; Disabled for performance reasons, see update-dashboard-card!-call-count-test
#_(derive :hook/search-index))
diff --git a/src/metabase/dashboards/models/dashboard_card_series.clj b/src/metabase/dashboards/models/dashboard_card_series.clj
index a2baf27d580..1bf369bd121 100644
--- a/src/metabase/dashboards/models/dashboard_card_series.clj
+++ b/src/metabase/dashboards/models/dashboard_card_series.clj
@@ -8,7 +8,8 @@
(methodical/defmethod t2/table-name :model/DashboardCardSeries [_model] :dashboardcard_series)
(doto :model/DashboardCardSeries
- (derive :metabase/model))
+ (derive :metabase/model)
+ (derive :hook/dependencies))
;; Serialization
diff --git a/src/metabase/dashboards/models/dashboard_tab.clj b/src/metabase/dashboards/models/dashboard_tab.clj
index abefe6676ef..424f5f5bdee 100644
--- a/src/metabase/dashboards/models/dashboard_tab.clj
+++ b/src/metabase/dashboards/models/dashboard_tab.clj
@@ -20,7 +20,8 @@
(derive ::mi/read-policy.full-perms-for-perms-set)
(derive ::mi/write-policy.full-perms-for-perms-set)
(derive :hook/timestamped?)
- (derive :hook/entity-id))
+ (derive :hook/entity-id)
+ (derive :hook/dependencies))
(methodical/defmethod t2/model-for-automagic-hydration [:metabase.dashboards.models.dashboard-card/DashboardCard :dashboard_tab]
[_original-model _k]
diff --git a/src/metabase/dependencies/hooks.clj b/src/metabase/dependencies/hooks.clj
new file mode 100644
index 00000000000..c7504438bd7
--- /dev/null
+++ b/src/metabase/dependencies/hooks.clj
@@ -0,0 +1,50 @@
+(ns metabase.dependencies.hooks
+ "Hooks that keep the dependency graph current as content is written, implemented by the enterprise module."
+ (:require
+ [metabase.premium-features.defenterprise :refer [defenterprise]]))
+
+(defenterprise before-insert
+ "Hook for the `model` row `instance` before it is inserted. The OSS implementation is a no-op."
+ metabase-enterprise.dependencies.hooks
+ [_model _instance]
+ nil)
+
+(defenterprise before-update
+ "Hook for the `model` row `instance` before it is updated, whose Toucan changes say what changes. The OSS
+ implementation is a no-op."
+ metabase-enterprise.dependencies.hooks
+ [_model _instance]
+ nil)
+
+(defenterprise before-delete
+ "Hook for the `model` row `instance` before it is deleted. The OSS implementation is a no-op."
+ metabase-enterprise.dependencies.hooks
+ [_model _instance]
+ nil)
+
+(defenterprise after-database-sync
+ "Hook for the Database with `database-id` after a sync. The OSS implementation is a no-op."
+ metabase-enterprise.dependencies.hooks
+ [_database-id]
+ nil)
+
+(defenterprise after-serdes-load
+ "Hook for after a serdes load, during which the hooks mark what the load changes without triggering the jobs. The
+ OSS implementation is a no-op."
+ metabase-enterprise.dependencies.hooks
+ []
+ nil)
+
+(defenterprise after-premium-token-update
+ "Hook for after the premium token changed, which may have enabled the dependencies feature. The OSS implementation is
+ a no-op."
+ metabase-enterprise.dependencies.hooks
+ []
+ nil)
+
+(defenterprise after-transform-run
+ "Hook for the transform with `transform-id` after a run wrote its output Table with `table-id`, or nil when the
+ table was not synced. The OSS implementation is a no-op."
+ metabase-enterprise.dependencies.hooks
+ [_transform-id _table-id]
+ nil)
diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.clj
index 266ee63b7d2..d0e92040fce 100644
--- a/src/metabase/documents/models/document.clj
+++ b/src/metabase/documents/models/document.clj
@@ -48,7 +48,8 @@
(derive :metabase/model)
(derive :perms/use-parent-collection-perms)
(derive :hook/timestamped?)
- (derive :hook/entity-id))
+ (derive :hook/entity-id)
+ (derive :hook/dependencies))
(defonce ^{:doc "Predicate gating a document's *content* (not merely its existence) below
collection-read, for documents whose rendered body embeds data the viewer may not
diff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.clj
index 318e30ecfb1..6e7153ee9b5 100644
--- a/src/metabase/measures/models/measure.clj
+++ b/src/metabase/measures/models/measure.clj
@@ -56,7 +56,8 @@
(doto :model/Measure
(derive :metabase/model)
(derive :hook/timestamped?)
- (derive :hook/entity-id))
+ (derive :hook/entity-id)
+ (derive :hook/dependencies))
(defmethod mi/can-read? :model/Measure
([instance]
diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.clj
index ddb68b6f6c4..8c56d7a6f94 100644
--- a/src/metabase/models/interface.clj
+++ b/src/metabase/models/interface.clj
@@ -18,6 +18,7 @@
[malli.transform :as mtx]
[medley.core :as m]
[metabase.app-db.core :as mdb]
+ [metabase.dependencies.hooks :as dependencies.hooks]
;; Toucan out-transforms normalize stored legacy MBQL on read; needed until the app db is MBQL 5
^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize]
;; stored card queries/refs are still legacy MBQL; validated against the legacy schema on read/write
@@ -38,10 +39,14 @@
[metabase.util.string :as string]
[methodical.core :as methodical]
[potemkin :as p]
+ [toucan2.connection :as t2.conn]
[toucan2.core :as t2]
[toucan2.model :as t2.model]
+ [toucan2.pipeline :as t2.pipeline]
[toucan2.protocols :as t2.protocols]
+ [toucan2.tools.before-delete :as t2.before-delete]
[toucan2.tools.before-insert :as t2.before-insert]
+ [toucan2.tools.before-update :as t2.before-update]
[toucan2.tools.hydrate :as t2.hydrate]
[toucan2.util :as t2.u])
(:import
@@ -676,6 +681,38 @@
(methodical/prefer-method! #'t2.before-insert/before-insert :hook/timestamped? :hook/entity-id)
(methodical/prefer-method! #'t2.before-insert/before-insert :hook/updated-at-timestamped? :hook/entity-id)
(methodical/prefer-method! #'t2.before-insert/before-insert :hook/created-at-timestamped? :hook/entity-id)
+
+(t2/define-before-insert :hook/dependencies
+ [instance]
+ (dependencies.hooks/before-insert &model instance)
+ instance)
+
+(t2/define-before-update :hook/dependencies
+ [instance]
+ (dependencies.hooks/before-update &model instance)
+ instance)
+
+(methodical/defmethod t2.pipeline/transduce-query :around [:toucan.query-type/update.* :hook/dependencies :default]
+ "Run the before-update hooks of a `:hook/dependencies` model in the transaction of its UPDATE, which Toucan otherwise
+ opens only after them, so dependency marks commit together with the UPDATE and after-commit work sees it."
+ [rf query-type model parsed-args resolved-query]
+ (t2.conn/with-transaction [_conn nil {:nested-transaction-rule :ignore}]
+ (next-method rf query-type model parsed-args resolved-query)))
+
+(t2/define-before-delete :hook/dependencies
+ [instance]
+ instance)
+
+(methodical/defmethod t2.before-delete/before-delete :before :hook/dependencies
+ [model instance]
+ (dependencies.hooks/before-delete model instance)
+ instance)
+
+(doseq [hook [:hook/timestamped? :hook/created-at-timestamped? :hook/updated-at-timestamped? :hook/entity-id]]
+ (methodical/prefer-method! #'t2.before-insert/before-insert hook :hook/dependencies))
+
+(doseq [hook [:hook/timestamped? :hook/updated-at-timestamped?]]
+ (methodical/prefer-method! #'t2.before-update/before-update hook :hook/dependencies))
;; --- helper fns
(defn changes-with-pk
"The row merged with the changes in pre-update hooks.
diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.clj
index 042e8a1ff56..ea852ffd30c 100644
--- a/src/metabase/models/serialization.clj
+++ b/src/metabase/models/serialization.clj
@@ -1746,7 +1746,7 @@
export-visualizer-settings
export-pivot-table
export-timeline-events
- (update :column_settings export-column-settings))))
+ (m/update-existing :column_settings export-column-settings))))
(defn- import-viz-link-card
[settings]
@@ -1838,6 +1838,13 @@
(m/update-existing :timeline.selected_timeline_ids import-fks entity-id? :model/Timeline)
(m/update-existing :timeline.excluded_timeline_event_ids import-fks timeline-event-ref? :model/TimelineEvent)))
+(defn- import-viz-column-settings
+ "Import `:column_settings`, dropping the `nil` that older exports wrote for settings without any."
+ [settings]
+ (if (some? (:column_settings settings))
+ (update settings :column_settings import-column-settings)
+ (dissoc settings :column_settings)))
+
(defn import-visualization-settings
"Given an EDN value as exported by [[export-visualization-settings]], convert its portable `[db schema table field]`
references into Field IDs."
@@ -1850,7 +1857,7 @@
import-visualizer-settings
import-pivot-table
import-timeline-events
- (update :column_settings import-column-settings))))
+ import-viz-column-settings)))
(defn- viz-link-card-deps
[allow-int-ids? settings]
diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.clj
index 66d82484e4f..1d2e2255051 100644
--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj
+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj
@@ -25,7 +25,8 @@
(doto :model/NativeQuerySnippet
(derive :metabase/model)
(derive :hook/timestamped?)
- (derive :hook/entity-id))
+ (derive :hook/entity-id)
+ (derive :hook/dependencies))
;; TODO (Cam 2026-07-08) Change Native Query Snippets to store template tags as a list like we do in MBQL as of 63.
(t2/deftransforms :model/NativeQuerySnippet
diff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.clj
index baf803453e3..4f2e2c52d3b 100644
--- a/src/metabase/premium_features/token_check.clj
+++ b/src/metabase/premium_features/token_check.clj
@@ -18,7 +18,6 @@
[metabase.analytics-interface.core :as analytics]
[metabase.app-db.core :as app-db]
[metabase.config.core :as config]
- [metabase.events.core :as events]
[metabase.internal-stats.core :as internal-stats]
[metabase.premium-features.db :as premium-features.db]
[metabase.premium-features.defenterprise :refer [defenterprise]]
@@ -614,8 +613,6 @@
([checker token]
(-check-token checker token)))
-(events/derive! :event/set-premium-embedding-token :metabase/event)
-
(defn -set-premium-embedding-token!
"Setter for the [[metabase.premium-features.settings/token-status]] setting."
[new-value]
@@ -645,7 +642,7 @@
:response/keys #{:error-details}}))))
(log/info "Token is valid."))
(setting/set-value-of-type! :string :premium-embedding-token new-value)
- (events/publish-event! :event/set-premium-embedding-token {})
+ ((requiring-resolve 'metabase.dependencies.hooks/after-premium-token-update))
(catch Throwable e
(log/errorf "Error setting premium features token: %s" (ex-message e))
;; merge in error-details if present
diff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.clj
index 112ea8a0009..755b35237d9 100644
--- a/src/metabase/queries/core.clj
+++ b/src/metabase/queries/core.clj
@@ -65,7 +65,6 @@
update-card!
visible-metric-cards-where-clause]
[metabase.queries.models.card.metadata
- infer-metadata
infer-metadata-with-model-overrides
maybe-async-result-metadata
refresh-metadata
diff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.clj
index 439836d5269..882265f5fb2 100644
--- a/src/metabase/queries/models/card.clj
+++ b/src/metabase/queries/models/card.clj
@@ -136,7 +136,8 @@
;; You can read/write a Card if you can read/write its parent Collection
(derive :perms/use-parent-collection-perms)
(derive :hook/timestamped?)
- (derive :hook/entity-id))
+ (derive :hook/entity-id)
+ (derive :hook/dependencies))
(defn- parent-document-id
"The `document_id` of `card`, or `::not-adjudicable` when the instance carries neither the column
diff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.clj
index 8f097a9bee2..d141cec702f 100644
--- a/src/metabase/segments/models/segment.clj
+++ b/src/metabase/segments/models/segment.clj
@@ -78,7 +78,8 @@
(doto :model/Segment
(derive :metabase/model)
(derive :hook/timestamped?)
- (derive :hook/entity-id))
+ (derive :hook/entity-id)
+ (derive :hook/dependencies))
(defmethod mi/can-read? :model/Segment
([instance]
diff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.clj
index 1eabacb2d7a..49b4f4cfd27 100644
--- a/src/metabase/sync/sync_metadata.clj
+++ b/src/metabase/sync/sync_metadata.clj
@@ -8,6 +8,7 @@
4. Sync FKs and indexes
5. Sync Metabase Metadata table (`metabase.sync.sync-metadata.metabase-metadata`)"
(:require
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.sync.fetch-metadata :as fetch-metadata]
[metabase.sync.interface :as i]
[metabase.sync.sync-metadata.dbms-version :as sync-dbms-ver]
@@ -93,6 +94,7 @@
(if abort-cause
(sync-util/set-initial-database-sync-aborted! database abort-cause)
(sync-util/set-initial-database-sync-complete! database))
+ (dependencies.hooks/after-database-sync (:id database))
results))
(mu/defn sync-db-metadata!
diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.clj
index 27f63257cec..31f9ff7f845 100644
--- a/src/metabase/transforms/models/transform.clj
+++ b/src/metabase/transforms/models/transform.clj
@@ -34,7 +34,7 @@
(methodical/defmethod t2/table-name :model/Transform [_model] :transform)
-(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped?]]
+(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped? :hook/dependencies]]
(derive :model/Transform trait))
(defn- transform-readable?
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index a96803cf2cf..093888530da 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -8,6 +8,7 @@
[java-time.api :as t]
[metabase.app-db.cluster-lock :as cluster-lock]
[metabase.database-routing.core :as database-routing]
+ [metabase.dependencies.hooks :as dependencies.hooks]
[metabase.driver :as driver]
[metabase.events.core :as events]
[metabase.indexes.models.table-index :as table-index]
@@ -731,9 +732,10 @@
{:keys [publish-events?]
:or {publish-events? true}} opts
db-id (transforms-base.i/target-db-id transform)
- database (transforms-base.db/database db-id)]
+ database (transforms-base.db/database db-id)
+ table (sync-target! target database)]
;; Sync target table, set target_table_id on transform, and mark table as owned by this transform
- (when-let [table (sync-target! target database)]
+ (when table
(transforms-base.db/update-transform! (:id transform) {:target_table_id (:id table)})
(transforms-base.db/update-table! (:id table) {:transform_id (:id transform)}))
;; ANALYZE the target before the run is observable, so dependents don't plan on stale stats.
@@ -745,6 +747,7 @@
(log/warnf "refresh-table-stats! failed for %s.%s: %s" (:schema target) (:name target) (ex-message t))))
;; Publish event after sync so the table exists in AppDB.
(when publish-events?
+ (dependencies.hooks/after-transform-run (:id transform) (:id table))
(events/publish-event! :event/transform-run-complete
{:object {:db-id db-id
:transform-id (:id transform)
diff --git a/src/metabase/warehouse_schema/models/dimension.clj b/src/metabase/warehouse_schema/models/dimension.clj
index fc5117ae389..8b4e977f7b7 100644
--- a/src/metabase/warehouse_schema/models/dimension.clj
+++ b/src/metabase/warehouse_schema/models/dimension.clj
@@ -20,7 +20,8 @@
(doto :model/Dimension
(derive :metabase/model)
(derive :hook/entity-id)
- (derive :hook/timestamped?))
+ (derive :hook/timestamped?)
+ (derive :hook/dependencies))
(t2/deftransforms :model/Dimension
{:type mi/transform-keyword})
diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.clj
index e3e5b140732..6cffef7dd62 100644
--- a/src/metabase/warehouse_schema/models/field_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj
@@ -28,7 +28,8 @@
(doto :model/FieldUserSettings
(derive :metabase/model)
- (derive :hook/timestamped?))
+ (derive :hook/timestamped?)
+ (derive :hook/dependencies))
(defn- with-set-flags
"Set the `_set` flag of every flagged column `effective` writes, unless `effective` sets the flag itself."
diff --git a/src/metabase/warehouse_schema/models/table_user_settings.clj b/src/metabase/warehouse_schema/models/table_user_settings.clj
index 63bdcdd39f2..da5d56f8986 100644
--- a/src/metabase/warehouse_schema/models/table_user_settings.clj
+++ b/src/metabase/warehouse_schema/models/table_user_settings.clj
@@ -34,7 +34,8 @@
(doto :model/TableUserSettings
(derive :metabase/model)
(derive :hook/timestamped?)
- (derive :hook/search-index))
+ (derive :hook/search-index)
+ (derive :hook/dependencies))
(methodical/defmethod t2/primary-keys :model/TableUserSettings [_model] [:table_id])
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 47f58049e96..ff83a1fe0d0 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -69,6 +69,7 @@
(doto :model/Database
(derive :metabase/model)
(derive :hook/timestamped?)
+ (derive :hook/dependencies)
;; Deliberately **not** deriving from `:hook/entity-id` because we should not be randomizing the `entity_id`s on
;; databases, tables or fields. Since the sync process can create them in multiple instances, randomizing them would
;; cause duplication rather than good matching if the two instances are later linked by serdes.
@@ -164,16 +165,6 @@
[_db-id]
(mi/superuser?))
-(defenterprise mark-transforms-stale-on-database-delete!
- "Hook called from the `:model/Database` before-delete. Transforms whose `source_database_id`
- is about to be SET NULL by the FK action survive the delete (so the analyst can read the
- SQL/Python and rebuild against a new DB), but their dependency-analysis findings need to
- be re-run so they surface on `/monitor/dependency-diagnostics/broken`. OSS is a no-op
- (no dependencies module)."
- metabase-enterprise.dependencies.events
- [_db-id]
- nil)
-
(defn- can-write?
[db-id]
(or (some-> db-id db-id->router-db-id can-write?)
@@ -433,11 +424,6 @@
(t2/define-before-delete :model/Database
[{id :id, driver :engine, :as database}]
- ;; Mark transforms with this DB as source as stale so dependency-diagnostics re-analyzes
- ;; them after the row's gone. Must run BEFORE the FK SET NULL fires (which happens when
- ;; the database row is deleted below) so we can still resolve the affected transforms by
- ;; source_database_id.
- (mark-transforms-stale-on-database-delete! id)
(unschedule-tasks! database)
(secret/delete-orphaned-secrets! database)
(delete-database-fields! id)
diff --git a/test/metabase/dashboards/models/dashboard_card_test.clj b/test/metabase/dashboards/models/dashboard_card_test.clj
index 67348e63262..21267e24445 100644
--- a/test/metabase/dashboards/models/dashboard_card_test.clj
+++ b/test/metabase/dashboards/models/dashboard_card_test.clj
@@ -250,8 +250,8 @@
:size_x 3
:size_y 4
:series []}])
- ;; this is usually 10, but it can be 11 sometimes in CI for some reason
- (is (contains? #{10 11} (call-count)))))))))
+ ;; this is usually 13, but it can be 14 sometimes in CI for some reason
+ (is (contains? #{13 14} (call-count)))))))))
(deftest ^:parallel normalize-parameter-mappings-test
(testing "DashboardCard parameter mappings should get normalized when coming out of the DB"
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yaml b/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yaml
index fb6010b92ba..68df1e121db 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yaml
@@ -33,8 +33,7 @@ dashcards:
- CATEGORY
- null
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
- entity_id: iMNmMMABHvbbiVT3ufez6
card_id: UEWVldEEZ-3sYUjjN54qz
created_at: '2026-04-02T02:59:02.778668Z'
@@ -45,8 +44,7 @@ dashcards:
inline_parameters: []
parameter_mappings: []
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
- entity_id: RasgF0LvnoJlu7ModLrqv
card_id: dYD1j3pQtJdctg8duByUL
created_at: '2026-04-02T02:59:02.778668Z'
@@ -57,8 +55,7 @@ dashcards:
inline_parameters: []
parameter_mappings: []
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
- entity_id: NSyK1LJldomowKbt0HpuE
card_id: ciG63nSLHKRTImt5iWEte
created_at: '2026-04-02T02:59:02.778668Z'
@@ -69,7 +66,6 @@ dashcards:
inline_parameters: []
parameter_mappings: []
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
tabs: []
width: fixed
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yaml b/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yaml
index e095dc1e53b..eefda2f100c 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yaml
@@ -31,7 +31,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
text: Products in {{category}}
virtual_card:
display: heading
@@ -47,7 +46,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
text: |-
Showing products in the **{{category}}** category
with price above **{{min_price}}**.
@@ -83,7 +81,6 @@ dashcards:
- PRICE
- null
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
tabs: []
width: fixed
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yaml
index a016901964c..3f85c089696 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yaml
@@ -56,7 +56,6 @@ dashcards:
- card_id: dShCrdMnthlyCumul0004
position: 2
visualization_settings:
- column_settings: null
graph.show_values: true
graph.x_axis.title_text: Month
graph.y_axis.title_text: Revenue
@@ -76,8 +75,7 @@ dashcards:
- - template-tag
- min_total
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
- entity_id: f9pAJKJNxCtrS93uIN7F4
card_id: dShCrdTtlRevScalr0006
created_at: '2026-04-02T02:59:02.778668Z'
@@ -89,7 +87,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
scalar.field: Total Revenue
tabs: []
width: fixed
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yaml
index f9bf8edcc19..ec99e36c388 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yaml
@@ -31,8 +31,7 @@ dataset_query:
- ORDERS
lib/type: mbql.stage/mbql
lib/type: mbql/query
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yaml
index 1b93ceb1677..2e111c34dbf 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yaml
@@ -76,8 +76,7 @@ result_metadata:
lib/original-name: count
lib/source: source/aggregations
lib/source-column-alias: count
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yaml
index 8819b743f00..8bfedce93bf 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yaml
@@ -76,8 +76,7 @@ result_metadata:
lib/original-name: count
lib/source: source/aggregations
lib/source-column-alias: count
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yaml
index 33a1eacf59d..3a28abca692 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yaml
@@ -81,8 +81,7 @@ result_metadata:
lib/original-name: sum
lib/source: source/aggregations
lib/source-column-alias: sum
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yaml
index 9003027709f..a06383061b3 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yaml
@@ -19,8 +19,7 @@ dataset_query:
type: number
lib/type: mbql.stage/native
lib/type: mbql/query
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yaml
index 2c194750480..abbbf50c0a2 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yaml
@@ -38,8 +38,7 @@ result_metadata:
lib/original-name: sum
lib/source: source/aggregations
lib/source-column-alias: sum
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yaml b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yaml
index c4d04aa2ed0..3a45faaea4d 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yaml
@@ -36,8 +36,7 @@ dashcards:
- CREATED_AT
- null
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
- entity_id: uyqXk5DNhkAMTp4W1TD9-
card_id: dShCrdProdsTables0007
created_at: '2026-04-02T02:59:02.778668Z'
@@ -51,8 +50,7 @@ dashcards:
inline_parameters: []
parameter_mappings: []
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
tabs:
- created_at: '2026-04-02T02:59:02.778668Z'
entity_id: XcmNByL-bsDQ15jLHBUwr
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yaml b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yaml
index 12e5ffdb3cf..92de3037adb 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yaml
@@ -245,8 +245,7 @@ result_metadata:
lib/source: source/table-defaults
lib/source-column-alias: RATING
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yaml b/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yaml
index 6f17f573b4b..ff7d0169779 100644
--- a/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yaml
+++ b/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yaml
@@ -17,7 +17,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
text: Sales Overview
virtual_card:
display: heading
@@ -31,7 +30,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
text: |-
This dashboard provides an overview of sales metrics.
**Key metrics** include revenue, order count, and average order value.
@@ -48,7 +46,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
link:
url: https://metabase.com/docs
virtual_card:
@@ -63,7 +60,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
link:
entity:
id: Q_jD-f-9clKLFZ2TfUG2h
@@ -80,7 +76,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
iframe: <iframe src="https://example.com/embed" width="100%" height="400"></iframe>
virtual_card:
display: iframe
@@ -94,7 +89,6 @@ dashcards:
parameter_mappings: []
series: []
visualization_settings:
- column_settings: null
virtual_card:
display: placeholder
tabs: []
diff --git a/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yaml b/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yaml
index 963cfefab65..e13c857eb09 100644
--- a/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yaml
+++ b/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yaml
@@ -25,8 +25,7 @@ dataset_query:
- PRODUCTS
lib/type: mbql.stage/mbql
lib/type: mbql/query
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yaml b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yaml
index 8ecc04be211..ac36ab0ae9a 100644
--- a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yaml
+++ b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yaml
@@ -33,7 +33,6 @@ dashcards:
- CATEGORY
- null
series: []
- visualization_settings:
- column_settings: null
+ visualization_settings: {}
tabs: []
width: fixed
diff --git a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yaml b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yaml
index 5f6e74255a8..232f317d9fb 100644
--- a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yaml
+++ b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yaml
@@ -122,8 +122,7 @@ result_metadata:
lib/original-name: avg
lib/source: source/aggregations
lib/source-column-alias: avg
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yaml b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yaml
index bba364bf26d..87fde29ba14 100644
--- a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yaml
+++ b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yaml
@@ -28,8 +28,7 @@ dataset_query:
type: text
lib/type: mbql.stage/native
lib/type: mbql/query
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yaml b/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yaml
index 30f746bce93..6cd2a8a7f97 100644
--- a/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yaml
@@ -204,8 +204,7 @@ result_metadata:
lib/original-name: avg
lib/source: source/aggregations
lib/source-column-alias: avg
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yaml b/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yaml
index e0f95b1f883..db5c8e644c2 100644
--- a/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yaml
@@ -238,8 +238,7 @@ result_metadata:
lib/original-name: total_revenue
lib/source: source/aggregations
lib/source-column-alias: total_revenue
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yaml b/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yaml
index 231fa0885e6..5938541eb3f 100644
--- a/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yaml
@@ -227,8 +227,7 @@ result_metadata:
lib/source: source/table-defaults
lib/source-column-alias: RATING
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yaml b/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yaml
index 43a0be2f890..edfcd25de71 100644
--- a/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yaml
@@ -169,8 +169,7 @@ result_metadata:
lib/original-name: share
lib/source: source/aggregations
lib/source-column-alias: share
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yaml b/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yaml
index 22f54a0326b..1a540768cfe 100644
--- a/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yaml
@@ -466,8 +466,7 @@ result_metadata:
lib/source: source/expressions
lib/source-column-alias: Rating as Float
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yaml b/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yaml
index 6b3e0c36a68..7191894b9e6 100644
--- a/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yaml
@@ -204,8 +204,7 @@ result_metadata:
lib/original-name: percentile
lib/source: source/aggregations
lib/source-column-alias: percentile
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yaml b/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yaml
index c55eb7de448..d04e87aab07 100644
--- a/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yaml
@@ -162,8 +162,7 @@ result_metadata:
lib/original-name: Profit
lib/source: source/expressions
lib/source-column-alias: Profit
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yaml b/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yaml
index 9c837611c06..b76249c8bc2 100644
--- a/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yaml
@@ -428,8 +428,7 @@ result_metadata:
lib/source: source/table-defaults
lib/source-column-alias: CREATED_AT
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yaml b/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yaml
index e6ccefde6aa..9bdd0cbbd46 100644
--- a/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yaml
@@ -662,8 +662,7 @@ result_metadata:
lib/source: source/joins
lib/source-column-alias: STATE
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/math_functions.yaml b/test_resources/serialization_baseline/collections/main/queries/math_functions.yaml
index 52ad67c1a83..c549d185fc9 100644
--- a/test_resources/serialization_baseline/collections/main/queries/math_functions.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/math_functions.yaml
@@ -468,8 +468,7 @@ result_metadata:
lib/source: source/expressions
lib/source-column-alias: Price Log
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yaml b/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yaml
index c2e13b80849..9a278818933 100644
--- a/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yaml
@@ -54,8 +54,7 @@ result_metadata:
lib/original-name: count
lib/source: source/aggregations
lib/source-column-alias: count
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml
index 7617397f76f..186a2991b4e 100644
--- a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml
@@ -29,8 +29,7 @@ dataset_query:
type: snippet
lib/type: mbql.stage/native
lib/type: mbql/query
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yaml
index 088d5c613fd..faac565760d 100644
--- a/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yaml
@@ -56,8 +56,7 @@ dataset_query:
widget-type: date/range
lib/type: mbql.stage/native
lib/type: mbql/query
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yaml
index 6d7eb059102..97a28b6bb62 100644
--- a/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yaml
@@ -43,8 +43,7 @@ dataset_query:
type: text
lib/type: mbql.stage/native
lib/type: mbql/query
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/nested_query.yaml b/test_resources/serialization_baseline/collections/main/queries/nested_query.yaml
index 9e910fd0549..b6f176a10ab 100644
--- a/test_resources/serialization_baseline/collections/main/queries/nested_query.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/nested_query.yaml
@@ -121,8 +121,7 @@ result_metadata:
lib/source: source/previous-stage
lib/source-column-alias: sum
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yaml b/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yaml
index 65cea0637a3..1b5628c9337 100644
--- a/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yaml
@@ -416,8 +416,7 @@ result_metadata:
lib/source: source/table-defaults
lib/source-column-alias: CREATED_AT
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yaml b/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yaml
index c80398a533c..bd2280b382d 100644
--- a/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yaml
@@ -202,8 +202,7 @@ result_metadata:
lib/source: source/previous-stage
lib/source-column-alias: total_revenue
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/string_functions.yaml b/test_resources/serialization_baseline/collections/main/queries/string_functions.yaml
index ebdd187e5d9..a9fd17e8f35 100644
--- a/test_resources/serialization_baseline/collections/main/queries/string_functions.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/string_functions.yaml
@@ -547,8 +547,7 @@ result_metadata:
lib/source: source/expressions
lib/source-column-alias: Title From 5th Char
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yaml b/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yaml
index 1610bfb18fe..4ae50ccda2e 100644
--- a/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yaml
@@ -299,8 +299,7 @@ result_metadata:
lib/source: source/table-defaults
lib/source-column-alias: QUANTITY
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yaml b/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yaml
index ddf85153438..13d67cc49af 100644
--- a/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yaml
@@ -459,8 +459,7 @@ result_metadata:
lib/source: source/expressions
lib/source-column-alias: Today Date
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yaml b/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yaml
index 418f90de7a3..9d2508cb92d 100644
--- a/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yaml
@@ -702,8 +702,7 @@ result_metadata:
lib/source: source/expressions
lib/source-column-alias: Extracted Day of Week ISO
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/total_revenue.yaml b/test_resources/serialization_baseline/collections/main/queries/total_revenue.yaml
index d79141ee972..2c0b92553a3 100644
--- a/test_resources/serialization_baseline/collections/main/queries/total_revenue.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/total_revenue.yaml
@@ -38,8 +38,7 @@ result_metadata:
lib/original-name: sum
lib/source: source/aggregations
lib/source-column-alias: sum
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/url_functions.yaml b/test_resources/serialization_baseline/collections/main/queries/url_functions.yaml
index 039b60a0b52..25593e35252 100644
--- a/test_resources/serialization_baseline/collections/main/queries/url_functions.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/url_functions.yaml
@@ -535,8 +535,7 @@ result_metadata:
lib/source: source/expressions
lib/source-column-alias: Source Path
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yaml b/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yaml
index b8324b27065..5ae4a336044 100644
--- a/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yaml
+++ b/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yaml
@@ -105,8 +105,7 @@ result_metadata:
lib/original-name: offset
lib/source: source/aggregations
lib/source-column-alias: offset
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/source_model.yaml b/test_resources/serialization_baseline/collections/main/source_model.yaml
index d53a8979038..6c6440f1cdf 100644
--- a/test_resources/serialization_baseline/collections/main/source_model.yaml
+++ b/test_resources/serialization_baseline/collections/main/source_model.yaml
@@ -319,8 +319,7 @@ result_metadata:
lib/source: source/table-defaults
lib/source-column-alias: QUANTITY
qp/implicit-field?: true
-visualization_settings:
- column_settings: null
+visualization_settings: {}
card_schema: 24
serdes/meta:
- model: Card
diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yaml
index 54a9ae16704..bd630194036 100644
--- a/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yaml
+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yaml
@@ -442,7 +442,6 @@ result_metadata:
lib/source-column-alias: CREATED_AT
qp/implicit-field?: true
visualization_settings:
- column_settings: null
map.center_latitude: 39.5
map.center_longitude: -98.35
map.latitude_column: LATITUDE
diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yaml
index f69cdb17d03..7458f9a7fec 100644
--- a/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yaml
+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yaml
@@ -82,7 +82,6 @@ result_metadata:
lib/source: source/aggregations
lib/source-column-alias: sum
visualization_settings:
- column_settings: null
waterfall.decrease_color: '#EF8C8C'
waterfall.increase_color: '#88BF4D'
waterfall.show_total: true
diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yaml
index a20b62ae694..d4f193a3a24 100644
--- a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yaml
+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yaml
@@ -74,7 +74,6 @@ result_metadata:
lib/source: source/aggregations
lib/source-column-alias: count
visualization_settings:
- column_settings: null
pie.colors:
Doohickey: '#ED6E6E'
Gadget: '#88BF4D'
diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yaml
index ba255c9cfe9..58c38495132 100644
--- a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yaml
+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yaml
@@ -103,7 +103,6 @@ result_metadata:
lib/source: source/aggregations
lib/source-column-alias: count
visualization_settings:
- column_settings: null
graph.dimensions:
- CREATED_AT
- CATEGORY
diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yaml
index 91646a28645..c067e50e615 100644
--- a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yaml
+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yaml
@@ -39,7 +39,6 @@ result_metadata:
lib/source: source/aggregations
lib/source-column-alias: sum
visualization_settings:
- column_settings: null
gauge.segments:
- color: '#ED6E6E'
label: Low
diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yaml
index 0a054033de4..d5c11b88491 100644
--- a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yaml
+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yaml
@@ -82,7 +82,6 @@ result_metadata:
lib/source: source/aggregations
lib/source-column-alias: sum
visualization_settings:
- column_settings: null
graph.goal_label: Monthly Target
graph.goal_value: 50000
graph.label_values_frequency: fit
diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yaml
index 6bf2a12efac..b61cf639055 100644
--- a/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yaml
+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yaml
@@ -74,7 +74,6 @@ result_metadata:
lib/source: source/aggregations
lib/source-column-alias: count
visualization_settings:
- column_settings: null
funnel.dimension: CATEGORY
funnel.metric: count
funnel.type: funnel