The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Recalculate dependencies of entities loaded by serdes and remote sync (#84313)
Alexander Polyankinda6911ee1668
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 349d9e85848..f9c3a3d2a90 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -890,6 +890,11 @@:api #{metabase.database-routing.core}:uses #{premium-features}}+ dependencies+ {:team "Graphy"+ :api #{metabase.dependencies.hooks}+ :uses #{premium-features}}+documents{:team "UX West":api #{metabase.documents.api@@ -1749,6 +1754,7 @@metabase.models.visualization-settings}:uses #{apiapp-db+ dependenciesliblib.legacy-mbqllib.metadata@@ -2014,7 +2020,7 @@:uses #{analytics.interfaceanalytics.internal-statsapp-db- events+ dependenciessettingsstartuptask@@ -2827,6 +2833,7 @@app-dbaudit-appdatabase-routing+ dependenciesdrivereventsinterestingness@@ -3058,6 +3065,7 @@:model/TransformRun}:uses #{app-dbdatabase-routing+ dependenciesdrivereventslib@@ -3886,11 +3894,11 @@:api #{metabase-enterprise.dependencies.api metabase-enterprise.dependencies.init metabase-enterprise.dependencies.models.dependency}:uses #{apiapp-db+ batch-processingcollectionsdatabase-routingdocumentsdriver- eventsliblib.belib.metadata@@ -3918,13 +3926,20 @@:model-imports #{:model/Card:model/Collection:model/Dashboard+ :model/DashboardCard+ :model/DashboardCardSeries+ :model/DashboardTab+ :model/Database+ :model/Dimension:model/Document+ :model/Field:model/FieldUserSettings:model/Measure:model/NativeQuerySnippet:model/Sandbox:model/Segment:model/Table+ :model/TableUserSettings:model/Transform}}enterprise/dependencies.replacement@@ -4210,7 +4225,6 @@app-dbaudit-appdriver- eventsliblib.belib.legacy-mbql@@ -4409,7 +4423,7 @@app-dbappearancecollections- events+ dependenciesliblib.schemaloggerdiff --git a/e2e/test/scenarios/admin/datamodel/segments.cy.spec.ts b/e2e/test/scenarios/admin/datamodel/segments.cy.spec.tsindex 37c004b62f6..5f0336e190a 100644--- a/e2e/test/scenarios/admin/datamodel/segments.cy.spec.ts+++ b/e2e/test/scenarios/admin/datamodel/segments.cy.spec.ts@@ -391,7 +391,7 @@ describe("scenarios > admin > datamodel > segments", () => {H.activateToken("pro-self-hosted");cy.log("set up remote sync");H.setupGitSync();- H.configureGit("read-only");+ H.configureGitAndPullChanges("read-only");H.createLibrary();H.publishTables({ table_ids: [ORDERS_ID] });// Let's leave PEOPLE_ID unpublisheddiff --git a/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.ts b/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.tsindex 7bb298961c3..39b70689910 100644--- a/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.ts+++ b/e2e/test/scenarios/data-studio/data-studio-library.cy.spec.ts@@ -425,7 +425,7 @@ describe("scenarios > data studio > library", () => {describe("read-only mode", () => {beforeEach(() => {H.setupGitSync();- H.configureGit("read-only");+ H.configureGitAndPullChanges("read-only");H.createLibrary();});diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/async.clj b/enterprise/backend/src/metabase_enterprise/dependencies/async.cljdeleted file mode 100644index 8f24b0c3588..00000000000--- a/enterprise/backend/src/metabase_enterprise/dependencies/async.clj+++ /dev/null@@ -1,24 +0,0 @@-(ns metabase-enterprise.dependencies.async- "Single-threaded executor for dependency event work.-- The two expensive post-save handlers (`::check-card-dependents` and `::update-card-dependents-metadata`)- submit their work here so the HTTP response returns immediately. Tasks execute serially- to avoid concurrent writes to dependent cards."- (:import- (java.util.concurrent Callable Executors ExecutorService ThreadFactory)))--(set! *warn-on-reflection* true)--(defonce ^:private executor- (delay (Executors/newSingleThreadExecutor- (reify ThreadFactory- (newThread [_ r]- (doto (Thread. r)- (.setName "dependency-event-worker")- (.setDaemon true)))))))--(defn submit!- "Submit dependency work to the single-threaded executor. Tasks execute serially."- [f]- (let [task (bound-fn* f)]- (.submit ^ExecutorService @executor ^Callable task)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.cljindex f84a1df56b4..620d2c59c6e 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj@@ -12,7 +12,6 @@[metabase.lib.schema.id :as lib.schema.id][metabase.models.interface :as mi][metabase.permissions.core :as perms]- [metabase.queries.schema :as queries.schema][metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]@@ -590,12 +589,6 @@[card-ids :- [:sequential ::lib.schema.id/card]](t2/select [:model/Card :id :database_id] :id [:in card-ids]))-(mu/defn set-card-result-metadata!- "Set the result metadata of the Card with `card-id`, returning the number updated."- [card-id :- ::lib.schema.id/card- result-metadata :- [:maybe ::queries.schema/card.result-metadata]]- (t2/update! :model/Card card-id {:result_metadata result-metadata}))-(mu/defn tables"The Tables with `table-ids`."[table-ids :- [:set ::lib.schema.id/table]]@@ -606,12 +599,15 @@[table-ids :- [:sequential ::lib.schema.id/table]](t2/select [:model/Table :id :db_id] :id [:in table-ids] {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))-(mu/defn table-id-by-name- "The ID of the Table named `table-name` in `schema` of the Database with `db-id`, or nil."- [db-id :- ::lib.schema.id/database- schema :- [:maybe :string]- table-name :- :string]- (t2/select-one-fn :id :model/Table :db_id db-id :schema schema :name table-name {:from [(warehouse-schema-overlay/table-query {:user-settings? false})]}))+(mu/defn dashcard-dashboard-id :- [:maybe ::lib.schema.id/dashboard]+ "The `:dashboard_id` of the DashboardCard with `dashcard-id`, or nil."+ [dashcard-id :- ::lib.schema.id/dashcard]+ (t2/select-one-fn :dashboard_id :model/DashboardCard :id dashcard-id))++(mu/defn field-table-id :- [:maybe ::lib.schema.id/table]+ "The `:table_id` of the Field with `field-id`, or nil."+ [field-id :- ::lib.schema.id/field]+ (t2/select-one-fn :table_id :model/Field :id field-id {:from [(warehouse-schema-overlay/field-query {:user-settings? false})]}))(mu/defn transform-sources"The `:id` and `:source` of the Transforms with `transform-ids`."@@ -684,10 +680,10 @@:to_entity_type to-type :to_entity_id to-id))(mu/defn delete-dependencies-from!- "Delete the Dependencies of the entity `entity-type` `entity-id`, returning the number deleted."+ "Delete the Dependencies of the entities `entity-type` `entity-ids`, returning the number deleted."[entity-type :- EntityType- entity-id :- ms/PositiveInt]- (t2/delete! :model/Dependency :from_entity_type entity-type :from_entity_id entity-id))+ entity-ids :- [:sequential ms/PositiveInt]]+ (t2/delete! :model/Dependency :from_entity_type entity-type :from_entity_id [:in entity-ids]))(mu/defn downstream-table-ids-of-transform"The IDs of the Tables that depend on the Transform with `transform-id`."@@ -716,11 +712,25 @@entity-id :- ms/PositiveInt](t2/select-one :model/DependencyStatus :entity_type entity-type :entity_id entity-id))-(mu/defn delete-dependency-status!- "Delete the DependencyStatus of the entity `entity-type` `entity-id`, returning the number deleted."+(mu/defn delete-dependency-statuses!+ "Delete the DependencyStatuses of the entities `entity-type` `entity-ids`, returning the number deleted."[entity-type :- EntityType- entity-id :- ms/PositiveInt]- (t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id entity-id))+ entity-ids :- [:sequential ms/PositiveInt]]+ (t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id [:in entity-ids]))++(mu/defn deleted-entity-ids :- [:sequential ms/PositiveInt]+ "Up to `batch-size` IDs of deleted entities of the type `entity-type` that still have a DependencyStatus."+ [entity-type :- ::deps.dependency-types/dependency-types+ batch-size :- ms/PositiveInt]+ (let [table-name (t2/table-name (deps.dependency-types/dependency-type->model entity-type))]+ (vec (t2/select-fn-vec :entity_id :model/DependencyStatus+ {:where [:and+ [:= :dependency_status.entity_type (name entity-type)]+ [:not [:exists ^:allow-subquery+ {:select [1]+ :from [[table-name :entity]]+ :where [:= :entity.id :dependency_status.entity_id]}]]]+ :limit batch-size}))))(mu/defn mark-dependency-status-stale!"Mark the DependencyStatus of `entity-type` `entity-id` as stale for dependency recalculation, creating it if it@@ -793,7 +803,9 @@[:or[:is :dependency_status.next_retry_at nil][:<= :dependency_status.next_retry_at now]]]]- :order-by [[[:case [:= :dependency_status.stale true] [:inline 0] :else [:inline 1]]]]+ :order-by [[[:case+ [:or [:= :dependency_status.entity_id nil] [:= :dependency_status.stale true]] [:inline 0]+ :else [:inline 1]]]]:limit batch-size})));;; ------------------------------------------------ Analysis findings ------------------------------------------------diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj b/enterprise/backend/src/metabase_enterprise/dependencies/events.cljdeleted file mode 100644index 4e10be69122..00000000000--- a/enterprise/backend/src/metabase_enterprise/dependencies/events.clj+++ /dev/null@@ -1,332 +0,0 @@-(ns metabase-enterprise.dependencies.events- (:require- [metabase-enterprise.dependencies.db :as dependencies.db]- [metabase-enterprise.dependencies.findings :as deps.findings]- [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]- [metabase-enterprise.dependencies.models.dependency :as models.dependency]- [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]- [metabase-enterprise.dependencies.task.backfill :as task.backfill]- [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]- [metabase.events.core :as events]- [metabase.premium-features.core :as premium-features :refer [defenterprise]]- [metabase.util.log :as log]- [methodical.core :as methodical]))--;; ## Maintaining the dependency graph-;; The below listens for inserts, updates and deletes of cards, snippets and transforms in order to keep the-;; dependency graph up to date. Transform *runs* are also a trigger, since the transform's output table may be created-;; or changed at that point.-;;-;; Create/update handlers mark entities stale in dependency_status. The backfill task does the actual computation.--(defn- mark-stale-and-trigger!- "Mark an entity as stale in dependency_status and trigger the backfill job."- [entity-type entity-id]- (try- (deps.dependency-status/mark-stale! entity-type [entity-id])- (task.backfill/trigger-backfill-job!)- (catch Throwable e- (log/error "Failed to mark entity stale" {:entity-type entity-type :entity-id entity-id :error (ex-message e)}))))--;; ### Cards-(events/derive! ::card-deps :metabase/event)-(events/derive! :event/card-create ::card-deps)-(events/derive! :event/card-update ::card-deps)--(methodical/defmethod events/publish-event! ::card-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :card (:id object))))--(events/derive! ::card-delete :metabase/event)-(events/derive! :event/card-delete ::card-delete)--(methodical/defmethod events/publish-event! ::card-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :card (:id object))- (dependencies.db/delete-dependency-status! :card (:id object))))--;; ### Snippets-(events/derive! ::snippet-deps :metabase/event)-(events/derive! :event/snippet-create ::snippet-deps)-(events/derive! :event/snippet-update ::snippet-deps)--(methodical/defmethod events/publish-event! ::snippet-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :snippet (:id object))))--(events/derive! ::snippet-delete :metabase/event)-(events/derive! :event/snippet-delete ::snippet-delete)--(methodical/defmethod events/publish-event! ::snippet-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :snippet (:id object))- (dependencies.db/delete-dependency-status! :snippet (:id object))))--;; ### Transforms-(events/derive! ::transform-deps :metabase/event)-(events/derive! :event/create-transform ::transform-deps)-(events/derive! :event/update-transform ::transform-deps)--(methodical/defmethod events/publish-event! ::transform-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :transform (:id object))))--(events/derive! ::transform-delete :metabase/event)-(events/derive! :event/delete-transform ::transform-delete)--(methodical/defmethod events/publish-event! ::transform-delete- [_ {:keys [id]}]- ;; TODO: (Braden 09/18/2025) Shouldn't we be deleting the downstream deps for dead edges as well as upstream?- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :transform id)- (dependencies.db/delete-dependency-status! :transform id)))--;; On *executing* a transform, its (freshly synced) output table is made to depend on the transform.-;; (And if the target has changed, the old table's dep on the transform is dropped.)-;; The upstream deps of the transform are not touched - those change only when the transform is edited.-(events/derive! ::transform-run :metabase/event)-(events/derive! :event/transform-run-complete ::transform-run)--(defn- transform-table-deps! [{:keys [db-id output-schema output-table transform-id] :as _details}]- (let [;; output-table is a keyword like :my_schema/my_table- table-name (name output-table)]- (when-let [table-id (dependencies.db/table-id-by-name db-id output-schema table-name)]- (models.dependency/replace-dependencies! :table table-id {:transform #{transform-id}}))))--(methodical/defmethod events/publish-event! ::transform-run- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (transform-table-deps! object)))--;; ### Dashboards-(events/derive! ::dashboard-deps :metabase/event)-(events/derive! :event/dashboard-create ::dashboard-deps)-(events/derive! :event/dashboard-update ::dashboard-deps)--(methodical/defmethod events/publish-event! ::dashboard-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :dashboard (:id object))))--(events/derive! ::dashboard-delete :metabase/event)-(events/derive! :event/dashboard-delete ::dashboard-delete)--(methodical/defmethod events/publish-event! ::dashboard-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :dashboard (:id object))- (dependencies.db/delete-dependency-status! :dashboard (:id object))))--;; ### Documents-(events/derive! ::document-deps :metabase/event)-(events/derive! :event/document-create ::document-deps)-(events/derive! :event/document-update ::document-deps)--(methodical/defmethod events/publish-event! ::document-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :document (:id object))))--(events/derive! ::document-delete :metabase/event)-(events/derive! :event/document-delete ::document-delete)--(methodical/defmethod events/publish-event! ::document-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :document (:id object))- (dependencies.db/delete-dependency-status! :document (:id object))))--;; ### Sandboxes-(events/derive! ::sandbox-deps :metabase/event)-(events/derive! :event/sandbox-create ::sandbox-deps)-(events/derive! :event/sandbox-update ::sandbox-deps)--(methodical/defmethod events/publish-event! ::sandbox-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :sandbox (:id object))))--(events/derive! ::sandbox-delete :metabase/event)-(events/derive! :event/sandbox-delete ::sandbox-delete)--(methodical/defmethod events/publish-event! ::sandbox-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :sandbox (:id object))- (dependencies.db/delete-dependency-status! :sandbox (:id object))))--;; ### Segments-(events/derive! ::segment-deps :metabase/event)-(events/derive! :event/segment-create ::segment-deps)-(events/derive! :event/segment-update ::segment-deps)--(methodical/defmethod events/publish-event! ::segment-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :segment (:id object))))--(events/derive! ::segment-delete :metabase/event)-(events/derive! :event/segment-delete ::segment-delete)--(methodical/defmethod events/publish-event! ::segment-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :segment (:id object))- (dependencies.db/delete-dependency-status! :segment (:id object))))--;; ### Measures-(events/derive! ::measure-deps :metabase/event)-(events/derive! :event/measure-create ::measure-deps)-(events/derive! :event/measure-update ::measure-deps)--(methodical/defmethod events/publish-event! ::measure-deps- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (mark-stale-and-trigger! :measure (:id object))))--(events/derive! ::measure-delete :metabase/event)-(events/derive! :event/measure-delete ::measure-delete)--(methodical/defmethod events/publish-event! ::measure-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (dependencies.db/delete-dependencies-from! :measure (:id object))- (dependencies.db/delete-dependency-status! :measure (:id object))))--;; ## Checking dependents for breakage (analysis_finding staleness)-;;-;; This is a SEPARATE staleness system from the dependency_status table above.-;; - dependency_status.stale: tracks whether an entity's upstream *dependency graph* needs recomputation.-;; Handled by the backfill job (task.backfill).-;; - analysis_finding staleness: tracks whether an entity's dependents need re-analysis for *breakage detection*-;; (e.g., a model column was removed — are downstream cards broken?). Handled by the entity-check job-;; (task.entity-check).-;;-;; Both are triggered from the same entity events but serve different purposes and run independently.--(events/derive! ::check-card-dependents :metabase/event)-(events/derive! :event/card-create ::check-card-dependents)-(events/derive! :event/card-update ::check-card-dependents)--(methodical/defmethod events/publish-event! ::check-card-dependents- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (deps.findings/mark-entity-and-transitive-dependents-stale! :card (:id object))- (task.entity-check/trigger-entity-check-job!)))--(events/derive! ::check-card-dependents-on-delete :metabase/event)-(events/derive! :event/card-delete ::check-card-dependents-on-delete)--(methodical/defmethod events/publish-event! ::check-card-dependents-on-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (when (deps.findings/mark-transitive-dependents-stale! {:card [(:id object)]})- (task.entity-check/trigger-entity-check-job!))))--(events/derive! ::check-transform :metabase/event)-(events/derive! :event/create-transform ::check-transform)-(events/derive! :event/update-transform ::check-transform)--(methodical/defmethod events/publish-event! ::check-transform- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (deps.findings/mark-entity-and-transitive-dependents-stale! :transform (:id object))- (task.entity-check/trigger-entity-check-job!)))--(events/derive! ::check-transform-on-delete :metabase/event)-(events/derive! :event/delete-transform ::check-transform-on-delete)--(methodical/defmethod events/publish-event! ::check-transform-on-delete- [_ {:keys [id]}]- (when (premium-features/has-feature? :dependencies)- (when (deps.findings/mark-transitive-dependents-stale! {:transform [id]})- (task.entity-check/trigger-entity-check-job!))))--(events/derive! ::check-segment-dependents :metabase/event)-(events/derive! :event/segment-create ::check-segment-dependents)-(events/derive! :event/segment-update ::check-segment-dependents)--(methodical/defmethod events/publish-event! ::check-segment-dependents- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (deps.findings/mark-entity-and-transitive-dependents-stale! :segment (:id object))- (task.entity-check/trigger-entity-check-job!)))--(events/derive! ::check-segment-dependents-on-delete :metabase/event)-(events/derive! :event/segment-delete ::check-segment-dependents-on-delete)--(methodical/defmethod events/publish-event! ::check-segment-dependents-on-delete- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (when (deps.findings/mark-transitive-dependents-stale! {:segment [(:id object)]})- (task.entity-check/trigger-entity-check-job!))))--(events/derive! ::check-transform-dependents :metabase/event)-(events/derive! :event/transform-run-complete ::check-transform-dependents)--(methodical/defmethod events/publish-event! ::check-transform-dependents- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (when (deps.findings/mark-transitive-dependents-stale! {:transform [(:transform-id object)]})- (task.entity-check/trigger-entity-check-job!))))--(defn- synced-db->direct-dependents-of-changed-tables- "Given the `:db_id` of a freshly synced database, this examines all tables in the DB which were updated, or have- fields which were updated, since the last time any cards depending on them were analyzed.-- It is important that this doesn't re-run the analysis for all dependents of every table whose DB got synced -- most of the tables have no change every time.-- Returns the set of table IDs which have dependents that need re-analysis, possibly empty."- [db-id]- (dependencies.db/table-ids-with-outdated-findings db-id))--(events/derive! ::sync-completed-on-database :metabase/event)-(events/derive! :event/sync-end ::sync-completed-on-database)--(methodical/defmethod events/publish-event! ::sync-completed-on-database- [_ {db-id :database_id}]- (when (premium-features/has-feature? :dependencies)- (let [changes (synced-db->direct-dependents-of-changed-tables db-id)]- (when (and (seq changes)- (deps.findings/mark-transitive-dependents-stale! {:table changes}))- (task.entity-check/trigger-entity-check-job!)))))--;; ### Admin UI Table/Field Metadata Updates-;; When a table or field's metadata is updated via the admin UI, re-analyze all dependents of that table.-(events/derive! ::check-table-metadata-update :metabase/event)-(events/derive! :event/table-update ::check-table-metadata-update)--(methodical/defmethod events/publish-event! ::check-table-metadata-update- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (when (deps.findings/mark-transitive-dependents-stale! {:table [(:id object)]})- (task.entity-check/trigger-entity-check-job!))))--(events/derive! ::check-field-metadata-update :metabase/event)-(events/derive! :event/field-update ::check-field-metadata-update)--(methodical/defmethod events/publish-event! ::check-field-metadata-update- [_ {:keys [object]}]- (when (premium-features/has-feature? :dependencies)- (when (deps.findings/mark-transitive-dependents-stale! {:table [(:table_id object)]})- (task.entity-check/trigger-entity-check-job!))))--;; ### Database Deletion (orphans transforms)-;; Transforms whose `source_database_id` matches the database being deleted survive the delete-;; but their existing `analysis_finding` rows still say "OK". Mark them stale here so the entity-check-;; job re-runs and surfaces them on `/monitor/dependency-diagnostics/broken`.-(defenterprise mark-transforms-stale-on-database-delete!- "Enterprise implementation: mark all transforms whose source database is being deleted as stale- for dependency re-analysis. See the OSS declaration in `metabase.warehouses.models.database`."- :feature :dependencies- [db-id]- (when-let [transform-ids (not-empty (dependencies.db/transform-ids-of-source-database db-id))]- (deps.analysis-finding/mark-stale! :transform transform-ids)- (deps.findings/mark-transitive-dependents-stale! {:transform transform-ids})- (task.entity-check/trigger-entity-check-job!)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj b/enterprise/backend/src/metabase_enterprise/dependencies/findings.cljindex 4bedc588254..990c09a3d52 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/findings.clj@@ -121,8 +121,7 @@(into {}(filter (fn [[dep-type _]] (analyzable-entities dep-type)))pass-through-dependents))]- (doseq [[dep-type dep-ids] all-supported]- (deps.analysis-finding/mark-stale! dep-type dep-ids))+ (deps.analysis-finding/mark-stale! all-supported)(boolean (seq all-supported))))(defn mark-transitive-dependents-stale!@@ -142,7 +141,7 @@"Mark a single entity as stale for re-analysis by the background job."[entity-type :- AnalyzableEntityTypeentity-id :- pos-int?]- (deps.analysis-finding/mark-stale! entity-type [entity-id]))+ (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))(mu/defn mark-entity-and-transitive-dependents-stale!"Mark `entity-id` and all of its transitive dependents stale for re-analysis. Use this when an entity changes: it@@ -162,7 +161,7 @@This only *analyzes* (each [[upsert-analysis!]] clears that entity's stale flag); it does NOT propagate staleness todependents. Propagation happens up front when an entity changes (see [[mark-entity-and-transitive-dependents-stale!]] and the- event handlers), so the entity-check job drains a fixed set of stale entities that only shrinks — guaranteeing+ backfill job), so the entity-check job drains a fixed set of stale entities that only shrinks — guaranteeingtermination even when the dependency graph has cycles."[type :- AnalyzableEntityTypebatch-size :- pos-int?]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.cljnew file mode 100644index 00000000000..69cd1891d49--- /dev/null+++ b/enterprise/backend/src/metabase_enterprise/dependencies/hooks.clj@@ -0,0 +1,239 @@+(ns metabase-enterprise.dependencies.hooks+ "Implements the hooks of `metabase.dependencies.hooks`, marking written content for the background jobs to recompute."+ (:require+ [metabase-enterprise.dependencies.db :as dependencies.db]+ [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.dependencies.findings :as deps.findings]+ [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]+ [metabase-enterprise.dependencies.models.dependency :as models.dependency]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]+ [metabase-enterprise.dependencies.task.backfill :as task.backfill]+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check]+ [metabase.app-db.core :as mdb]+ [metabase.batch-processing.core :as grouper]+ [metabase.models.interface :as mi]+ [metabase.premium-features.core :refer [defenterprise]]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- trigger-jobs-now!+ "Trigger the backfill and entity-check jobs, logging rather than throwing a failure."+ []+ (try+ (task.backfill/trigger-backfill-job!)+ (task.entity-check/trigger-entity-check-job!)+ (catch Throwable e+ (log/warnf "Failed to trigger the dependency jobs: %s" (ex-message e)))))++(defn- process-writes!+ "Mark the dependents of the Tables changed by `writes` for re-analysis, then trigger the jobs."+ [writes]+ (when-let [table-ids (not-empty (into #{} (keep :table-id) writes))]+ (deps.findings/mark-transitive-dependents-stale! {:table table-ids}))+ (trigger-jobs-now!))++(def ^:private writes-queue+ (delay (grouper/start! process-writes! :capacity 500 :interval 1000)))++(defn- submit-after-commit!+ "Submit `write` to the batch processed once the current transaction commits."+ [write]+ (mdb/do-after-commit #(grouper/submit! @writes-queue write)))++(defn- trigger-jobs!+ "Have the backfill and entity-check jobs triggered for this write, once it commits, together with the others in its+ batch, unless a serdes load triggers them once it finishes."+ []+ (when-not mi/*deserializing?*+ (submit-after-commit! :ok)))++(defn- mark-changed!+ "Mark the entity `entity-type` `entity-id` for recalculation and, when analyzable, for re-analysis; the backfill job+ marks its dependents for re-analysis."+ [entity-type entity-id]+ (when entity-id+ (deps.dependency-status/mark-stale! {entity-type #{entity-id}})+ (when (contains? deps.findings/analyzable-entities entity-type)+ (deps.analysis-finding/mark-stale! {entity-type #{entity-id}}))+ (trigger-jobs!)))++(defn- changed?+ "Whether the Toucan update of `instance` changes any of `columns`."+ [instance columns]+ (let [changes (t2/changes instance)]+ (some #(contains? changes %) columns)))++(defn- mark-entity-changed!+ "Mark the `model` entity `instance` for recalculation when its update changes any of `columns`."+ [model instance columns]+ (when (changed? instance columns)+ (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))++(defmulti ^:private before-insert*+ "Mark what inserting the `model` row `instance` affects."+ {:arglists '([model instance])}+ (fn [model _instance] model))++(defmulti ^:private before-update*+ "Mark what updating the `model` row `instance` affects."+ {:arglists '([model instance])}+ (fn [model _instance] model))++(defmulti ^:private before-delete*+ "Mark what deleting the `model` row `instance` affects."+ {:arglists '([model instance])}+ (fn [model _instance] model))++(defmethod before-insert* :default [_model _instance] nil)+(defmethod before-update* :default [_model _instance] nil)+(defmethod before-delete* :default [_model _instance] nil)++(doseq [model [:model/Card :model/Dashboard :model/Document :model/Measure :model/NativeQuerySnippet :model/Sandbox+ :model/Segment :model/Transform]]+ (defmethod before-insert* model+ [_model _instance]+ (trigger-jobs!))+ (defmethod before-delete* model+ [model instance]+ (mark-changed! (deps.dependency-types/model->dependency-type model) (:id instance))))++(doseq [[model columns] {:model/Card #{:type :dataset_query :dimension_mappings :parameters}+ :model/Dashboard #{:parameters}+ :model/Document #{:content_type :document}+ :model/Measure #{:definition :table_id}+ :model/NativeQuerySnippet #{:content :template_tags}+ :model/Sandbox #{:card_id}+ :model/Segment #{:definition :table_id}+ :model/Transform #{:source :target}}]+ (defmethod before-update* model+ [model instance]+ (mark-entity-changed! model instance columns)))++(defmethod before-insert* :model/DashboardCard+ [_model dashcard]+ (mark-changed! :dashboard (:dashboard_id dashcard)))++(defmethod before-update* :model/DashboardCard+ [_model dashcard]+ (when (changed? dashcard #{:card_id :dashboard_id :visualization_settings})+ (doseq [dashboard-id (distinct [(:dashboard_id (t2/original dashcard)) (:dashboard_id dashcard)])]+ (mark-changed! :dashboard dashboard-id))))++(defmethod before-delete* :model/DashboardCard+ [_model dashcard]+ (mark-changed! :dashboard (:dashboard_id dashcard)))++(defmethod before-insert* :model/DashboardCardSeries+ [_model series]+ (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))++(defmethod before-update* :model/DashboardCardSeries+ [_model series]+ (when (changed? series #{:card_id :dashboardcard_id})+ (doseq [dashcard-id (distinct [(:dashboardcard_id (t2/original series)) (:dashboardcard_id series)])]+ (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id dashcard-id)))))++(defmethod before-delete* :model/DashboardCardSeries+ [_model series]+ (mark-changed! :dashboard (dependencies.db/dashcard-dashboard-id (:dashboardcard_id series))))++(defmethod before-delete* :model/DashboardTab+ [_model tab]+ (mark-changed! :dashboard (:dashboard_id tab)))++(defn- mark-table-changed!+ "Mark the dependents of the Table with `table-id` for re-analysis once the current transaction commits."+ [table-id]+ (when table-id+ (submit-after-commit! {:table-id table-id})))++(defmethod before-insert* :model/TableUserSettings+ [_model settings]+ (mark-table-changed! (:table_id settings)))++(defmethod before-update* :model/TableUserSettings+ [_model settings]+ (mark-table-changed! (:table_id settings)))++(defmethod before-delete* :model/TableUserSettings+ [_model settings]+ (mark-table-changed! (:table_id settings)))++(defmethod before-insert* :model/FieldUserSettings+ [_model settings]+ (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))++(defmethod before-update* :model/FieldUserSettings+ [_model settings]+ (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))++(defmethod before-delete* :model/FieldUserSettings+ [_model settings]+ (mark-table-changed! (dependencies.db/field-table-id (:field_id settings))))++(defmethod before-insert* :model/Dimension+ [_model dimension]+ (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))++(defmethod before-update* :model/Dimension+ [_model dimension]+ (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))++(defmethod before-delete* :model/Dimension+ [_model dimension]+ (mark-table-changed! (dependencies.db/field-table-id (:field_id dimension))))++(defmethod before-delete* :model/Database+ [_model database]+ (doseq [transform-id (dependencies.db/transform-ids-of-source-database (:id database))]+ (mark-changed! :transform transform-id)))++(defenterprise before-insert+ "Mark what inserting the `model` row `instance` affects for recalculation."+ :feature :dependencies+ [model instance]+ (before-insert* model instance))++(defenterprise before-update+ "Mark what updating the `model` row `instance` affects for recalculation."+ :feature :dependencies+ [model instance]+ (before-update* model instance))++(defenterprise before-delete+ "Mark what deleting the `model` row `instance` affects for recalculation."+ :feature :dependencies+ [model instance]+ (before-delete* model instance))++(defenterprise after-database-sync+ "Mark the dependents of the Tables of the Database with `database-id` that changed since they were analyzed."+ :feature :dependencies+ [database-id]+ (when (deps.findings/mark-transitive-dependents-stale!+ {:table (dependencies.db/table-ids-with-outdated-findings database-id)})+ (trigger-jobs!)))++(defenterprise after-transform-run+ "Make the output Table with `table-id` depend on the transform with `transform-id`, and mark the transform's+ dependents for re-analysis."+ :feature :dependencies+ [transform-id table-id]+ (when table-id+ (models.dependency/replace-dependencies! :table table-id {:transform #{transform-id}}))+ (when (deps.findings/mark-transitive-dependents-stale! {:transform #{transform-id}})+ (trigger-jobs!)))++(defenterprise after-serdes-load+ "Trigger the backfill and entity-check jobs for what a serdes load marked."+ :feature :dependencies+ []+ (trigger-jobs-now!))++(defenterprise after-premium-token-update+ "Restart the backfill and entity-check jobs, which stop while the dependencies feature is disabled."+ :feature :dependencies+ []+ (trigger-jobs-now!))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj b/enterprise/backend/src/metabase_enterprise/dependencies/init.cljindex 0980db3bfd7..29087660a17 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/init.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/init.clj@@ -1,7 +1,6 @@(ns metabase-enterprise.dependencies.init(:require- [metabase-enterprise.dependencies.events]- [metabase-enterprise.dependencies.metadata-update]+ [metabase-enterprise.dependencies.hooks][metabase-enterprise.dependencies.schema][metabase-enterprise.dependencies.settings][metabase-enterprise.dependencies.task.backfill]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.cljindex 33343dc604a..55815a561c6 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_provider.clj@@ -78,7 +78,7 @@(declare all-overrides setup-transform! setup-transforms!)-(deftype OverridingMetadataProvider [delegate *overrides returned-columns-fn]+(deftype OverridingMetadataProvider [delegate *overrides]lib.metadata.protocols/MetadataProvider(database [_this] (lib.metadata.protocols/database delegate))(metadatas [this metadata-spec]@@ -121,25 +121,18 @@(assoc-in m ks v))% kvs)))-(defn- returned-columns [^OverridingMetadataProvider mp queryable]- (let [returned-columns-fn (.returned-columns-fn mp)]- (or (and returned-columns-fn (returned-columns-fn mp queryable))- (deps.analysis/returned-columns (:engine (lib.metadata/database mp))- (lib/query mp queryable)))))+(defn- returned-columns [mp queryable]+ (deps.analysis/returned-columns (:engine (lib.metadata/database mp))+ (lib/query mp queryable)))(defmethod add-override :card [^OverridingMetadataProvider mp _entity-type id updates](with-overrides mp- ;; If the `updates` contain `:result-metadata`, we want to use that. Similarly, if the user provides a way to- ;; calculate `result-metadata`, we should use that function. However, any `:result-metadata` from the inner-mp- ;; should be ignored.- {[:metadata/card id] (delay (let [temp (merge (when id- (-> (inner-mp mp)- (lib.metadata.protocols/card id)))- updates)- result-metadata (or (:result-metadata updates)- (and (.returned-columns-fn mp)- (returned-columns mp temp)))]- (assoc temp :result-metadata result-metadata)))+ ;; Only `:result-metadata` from the `updates` is used; any from the inner-mp is ignored.+ {[:metadata/card id] (delay (-> (merge (when id+ (-> (inner-mp mp)+ (lib.metadata.protocols/card id)))+ updates)+ (assoc :result-metadata (:result-metadata updates))));; This uses the outer OMP and so the overrides are visible![::card-columns id] (delay (returned-columns mp (lib.metadata/card mp id)))}))@@ -282,10 +275,8 @@Important note: all overrides must be added first, before this is used as a `MetadataProvider`. The easiest way todo that is to pass in updated entities and/or dependent ids, which will be immediately added as overrides."- [{:keys [base-provider updated-entities dependent-ids returned-columns-fn]}]- (let [^OverridingMetadataProvider omp (->OverridingMetadataProvider base-provider- (atom {})- returned-columns-fn)]+ [{:keys [base-provider updated-entities dependent-ids]}]+ (let [^OverridingMetadataProvider omp (->OverridingMetadataProvider base-provider (atom {}))](doseq [[entity-type updates] updated-entitiesupdated-entity updates](add-override omp entity-type (:id updated-entity) updated-entity))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj b/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.cljdeleted file mode 100644index 2e41ede9bb8..00000000000--- a/enterprise/backend/src/metabase_enterprise/dependencies/metadata_update.clj+++ /dev/null@@ -1,209 +0,0 @@-(ns metabase-enterprise.dependencies.metadata-update- (:require- [clojure.core.cache :as cache]- [medley.core :as m]- [metabase-enterprise.dependencies.async :as async]- [metabase-enterprise.dependencies.db :as dependencies.db]- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]- [metabase-enterprise.dependencies.metadata-provider :as deps.metadata-provider]- [metabase-enterprise.dependencies.models.dependency :as models.dependency]- [metabase.events.core :as events]- [metabase.graph.core :as graph]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.lib.schema.common :as lib.schema.common]- [metabase.lib.schema.id :as lib.schema.id]- [metabase.lib.schema.metadata :as lib.schema.metadata]- [metabase.premium-features.core :as premium-features]- [metabase.queries.core :as queries]- [metabase.queries.schema :as queries.schema]- [metabase.util :as u]- [metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]- [methodical.core :as methodical]))--(defn- mbql-graph- "Returns a graph that is limited to sandboxes, segments, measures, and mbql cards.-- All other types of nodes are ignored and are neither included in the graph or traversed to find transitive- dependents."- [mp]- (-> (models.dependency/filtered-graph-dependents nil {:types #{:card :sandbox :segment :measure}})- (graph/filtered-graph (fn [[type id]]- (not (and (= type :card)- (->> (lib.metadata/card mp id)- (models.dependency/is-native-entity? :card))))))- graph/cached-graph))--(mu/defn- dependent-mbql-cards :- [:sequential ::lib.schema.id/card]- "Returns a list of all card dependencies in the transitive children of `[start-type start-id]` using `graph`."- [graph :- ::graph/graph- start-type :- ::deps.dependency-types/dependency-types- start-id :- ::deps.dependency-types/entity-id]- (let [start [start-type start-id]]- (->> (graph/transitive graph [[start-type start-id]])- (keep (fn [[node-type node-id :as node]]- (when (and (= node-type :card)- (not= node start))- node-id))))))--(mr/def ::column-metadata-edits- "Edits `card-metadata-edits` reports: the keys are QP-results-cased (plain keys snake_case, `:lib/...` keys stay- kebab-case), since values are looked up straight out of `qp-results-cased-col` maps."- [:map {:closed true}- [:id {:optional true} [:maybe ::lib.schema.id/field]]- [:description {:optional true} [:maybe :string]]- [:display_name {:optional true} [:maybe :string]]- [:semantic_type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]]- [:fk_target_field_id {:optional true} [:maybe ::lib.schema.id/field]]- [:settings {:optional true} [:maybe ::lib.schema.common/visualization-settings]]- [:visibility_type {:optional true} [:maybe ::lib.schema.metadata/column.visibility-type]]- [:lib/source-display-name {:optional true} [:maybe :string]]])-(mr/def ::card-metadata-edits [:map-of :string ::column-metadata-edits])-(mr/def ::card-list-metadata-edits [:map-of ::lib.schema.id/card ::card-metadata-edits])--(mu/defn- column-metadata-edits :- [:maybe ::column-metadata-edits]- [editable-keys :- [:sequential :keyword]- original :- [:maybe :metabase.query-processor.middleware.annotate/qp-results-cased-col]- calculated :- [:maybe :metabase.query-processor.middleware.annotate/qp-results-cased-col]]- (when (and original calculated)- (-> (into {}- (keep (fn [key]- (let [calculated-value (calculated key)- original-value (original key)]- (when (and calculated-value original-value- (not= calculated-value original-value))- [key original-value]))))- editable-keys)- not-empty)))--(mu/defn- card-metadata-edits :- [:maybe ::card-metadata-edits]- [mp :- ::lib.schema.metadata/metadata-providerable- card-id :- ::lib.schema.id/card]- (let [card (lib.metadata/card mp card-id)- original-metadata (:result-metadata card)- ;; Replace the metadata provider here to ensure that it is using the correct one.- calculated-metadata (->> card :dataset-query (lib/query mp) queries/infer-metadata)- calculated-by-name (m/index-by :lib/deduplicated-name calculated-metadata)- editable-keys (->> (lib/model-preserved-keys false)- (map u/->snake_case_en))]- (-> (into {}- (keep (fn [{name :lib/deduplicated-name :as original}]- (let [calculated (calculated-by-name name)]- (when-let [edits (column-metadata-edits editable-keys original calculated)]- [name edits]))))- original-metadata)- not-empty)))--(mu/defn- card-list-metadata-edits :- ::card-list-metadata-edits- "Takes a metadata provider and a list of card ids, and finds the differences between the actual metadata in the db- and the computed metadata we would expect to see."- [mp :- ::lib.schema.metadata/metadata-providerable- card-ids :- [:sequential ::lib.schema.id/card]]- (into {}- (keep (fn [card-id]- (when-let [edit-map (card-metadata-edits mp card-id)]- [card-id edit-map])))- card-ids))--(mu/defn- updated-metadata- "Given a metadata provider, card, and metadata edit map, calculate new metadata with the appropriate metadata edits.-- Takes a metadata provider because the queries' existing metadata provider may not be the one we want to use. The- parent function should be ensuring that the passed-in metadata provider is correct."- [mp :- ::lib.schema.metadata/metadata-providerable- card :- ::lib.schema.metadata/card- metadata-edits :- ::card-list-metadata-edits]- (let [new-metadata (->> card :dataset-query (lib/query mp) queries/infer-metadata)- card-edits (metadata-edits (:id card))]- (if (= (:type card) :model)- (for [{name :lib/deduplicated-name :as column} new-metadata]- (merge column (when card-edits (card-edits name))))- new-metadata)))--(mu/defn- fresh-mp :- ::lib.schema.metadata/metadata-provider- "Creates a 'fresh' metadata provider for the given database id with no existing cache."- [db-id :- ::lib.schema.id/database]- (lib-be/with-existing-metadata-provider-cache (atom (cache/basic-cache-factory {}))- (lib-be/application-database-metadata-provider db-id)))--(mu/defn- update-dependent-mbql-cards-metadata!- [original-mp :- ::lib.schema.metadata/metadata-providerable- start-type :- ::deps.dependency-types/dependency-types- start-id :- ::deps.dependency-types/entity-id- previous-object :- ::queries.schema/card- metadata-type :- :keyword]- ;; Notes on metadata providers:- ;;- ;; The key thing here is that lib.metadata/card caches very aggressively (as of 2026/01, at least). Instead of- ;; going through the normal cache mechanism, it grabs the card normally, normalizes the query, and then caches the- ;; result using a special key. The next time it gets called, it will check for that key first and never actually- ;; hit the standard card fetch flow at all. Meanwhile, override metadata providers delegate all caching to their- ;; underlying metadata provider. As a result, an override metadata provider and its underlying metadata provider- ;; effectively use the same card cache. Overriding the card doesn't affect that cache, because lib.metadata/card- ;; will effectively ask the underlying metadata provider for the card (using that special key) without ever talking- ;; to the override metadata provider. Both the override metadata provider and the underlying metadata provider will- ;; always return whichever version of that card was requested and cached first.- ;;- ;; The solution here is three different metadata providers. First, we have the original metadata provider (or- ;; really, metadata-providerable -- a query is fine here). This doesn't have any special overrides and is used to- ;; fetch the current state of the world before any updates that happen in this function.- ;;- ;; Second, we have a "pre-update" metadata provider. This is an override metadata provider based on a "fresh"- ;; metadata provider with no cached data. This has overrides intended to match the state of the world before- ;; whatever update triggered this function call.- ;;- ;; Third, we have a "updated" metadata provider. This another override metadata provider based on a fresh metadata- ;; provider, but this one has no initial overrides. Instead, it is set to use our returned-columns logic and asked- ;; to re-calculate all of the child cards' result metadata. In this case, caching actually works in our favor --- ;; the override metadata provider does ask its base provider for the original card, and this would cache the old- ;; version of the card. However, the override metadata provider does this during a call lib.metadata/card. When- ;; that "parent" call finishes, it stores the updated version of the card in the cache, and that overwrites the old- ;; version that was previously cached.- ;;- ;; If we ever move to a world where model metadata edits are stored separately, we could and should use those- ;; calculated edits instead of computing them from scratch. However, existing models still wouldn't have edits, so- ;; we'd need to keep the existing code around for those existing cards. A backfill job could help here, but we'd- ;; still need to account for cases where we just upgraded an instance and the backfill job hasn't finished yet.- (let [db-id (-> (lib.metadata/database original-mp)- :id)- graph (mbql-graph original-mp)- cards (dependent-mbql-cards graph start-type start-id)- previous-metadata (lib-be/instance->metadata previous-object metadata-type)- pre-update-mp (deps.metadata-provider/override-metadata-provider- {:base-provider (fresh-mp db-id)- :updated-entities {start-type [previous-metadata]}})- edits (card-list-metadata-edits pre-update-mp cards)- updated-mp (deps.metadata-provider/override-metadata-provider- {:base-provider (fresh-mp db-id)- :returned-columns-fn (fn [mp card]- (updated-metadata mp card edits))- :dependent-ids {:card cards}})- start [start-type start-id]- updates (->> (graph/transitive-children-of graph [start])- (graph/keep-children- (fn [[node-type node-id :as node]]- (when (and (= node-type :card)- (not= node start))- (let [new-metadata (-> (lib.metadata/card updated-mp node-id)- :result-metadata)- old-metadata (-> (lib.metadata/card original-mp node-id)- :result-metadata)]- (if (= new-metadata old-metadata)- ::graph/stop- [node-id new-metadata]))))))]- (doseq [[card-id new-metadata] updates]- (dependencies.db/set-card-result-metadata! card-id new-metadata))))--(events/derive! ::update-card-dependents-metadata :metabase/event)-(events/derive! :event/card-update ::update-card-dependents-metadata)--(methodical/defmethod events/publish-event! ::update-card-dependents-metadata- [_ {{:keys [id dataset_query]} :object :keys [previous-object]}]- (when (and (premium-features/has-feature? :dependencies)- (seq dataset_query)- (not (lib/any-native-stage? dataset_query)))- (async/submit!- (fn []- (update-dependent-mbql-cards-metadata! dataset_query :card id previous-object :metadata/card)))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.cljindex 792cb8f6586..0a3dfbbd278 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj@@ -76,10 +76,11 @@1000)(defn mark-stale!- "Mark entities of `entity-type` with ids in `entity-ids` and existing findings as stale for re-analysis.- Entities without existing findings are ignored - they'll be picked up by the normal job flow."- [entity-type entity-ids]- (doseq [batch (partition-all mark-stale-batch-size entity-ids)]+ "Mark the existing findings of the entities in `type->ids`, `{entity-type ids}`, stale for re-analysis. Entities+ without findings are ignored - they'll be picked up by the normal job flow."+ [type->ids]+ (doseq [[entity-type ids] type->ids+ batch (partition-all mark-stale-batch-size ids)](dependencies.db/mark-findings-stale! entity-type batch)))(defn has-stale-entities?diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.cljindex 0513a1660d2..20f69662c0b 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj@@ -16,11 +16,11 @@{:entity_type mi/transform-keyword})(defn mark-stale!- "Mark entities of `entity-type` with ids in `entity-ids` as stale for dependency recalculation.- Creates entries if they don't exist, or sets stale=true if they do.- Resets retry state so previously-failed entities get a fresh chance."- [entity-type entity-ids]- (doseq [id entity-ids]+ "Mark the entities in `type->ids`, `{entity-type ids}`, stale for dependency recalculation, creating their statuses+ if they don't exist and resetting their retry state so previously-failed entities get a fresh chance."+ [type->ids]+ (doseq [[entity-type ids] type->ids+ id ids](dependencies.db/mark-dependency-status-stale! entity-type id)))(defn upsert-status!@@ -60,12 +60,14 @@(defn has-stale-or-outdated?"Returns true if there are any entities needing dependency calculation: no status row yet,- stale=true, OR version < current (not terminal, retry delay elapsed). Defined in terms of- [[instances-for-dependency-calculation]] so it stays consistent with what the backfill processes."+ stale=true, OR version < current (not terminal, retry delay elapsed), or deleted entities whose status the backfill+ has yet to clean up. Defined in terms of [[instances-for-dependency-calculation]] so it stays consistent with what+ the backfill processes."[](boolean(some (fn [entity-type]- (seq (instances-for-dependency-calculation entity-type 1)))+ (or (seq (instances-for-dependency-calculation entity-type 1))+ (seq (dependencies.db/deleted-entity-ids entity-type 1))))deps.dependency-types/backfillable-dependency-types)))(defn record-failure!diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.cljindex 9278b76bdae..3e50587b6eb 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj@@ -5,7 +5,8 @@This is done by querying the dependency_status table for entities that are stale or have anoutdated dependency_analysis_version. The backfill task computes dependencies and updates- the dependency_status table."+ the dependency_status table. Entities that a write marked stale, or that were deleted, also have their dependents+ marked for re-analysis here, outside the write's transaction."(:require[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.scheduler :as qs]@@ -14,17 +15,17 @@[metabase-enterprise.dependencies.calculation :as deps.calculation][metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]+ [metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.dependency :as models.dependency][metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status][metabase-enterprise.dependencies.settings :as deps.settings][metabase-enterprise.dependencies.task-util :as deps.task-util]- [metabase.events.core :as events]+ [metabase-enterprise.dependencies.task.entity-check :as task.entity-check][metabase.lib-be.core :as lib-be][metabase.premium-features.core :as premium-features][metabase.task.core :as task][metabase.transforms.core :as transforms][metabase.util.log :as log]- [methodical.core :as methodical][toucan2.core :as t2])(:import (org.quartz JobExecutionContext)))@@ -75,16 +76,31 @@(deps.dependency-status/instances-for-dependency-calculation entity-type batch-size)))(defn- compute-deps-for-entity!- "Compute and store dependencies for an entity, then update its dependency_status.- Entities are expected to be pre-hydrated by [[deps.dependency-status/hydrate-for-deps]]."+ "Mark the dependents of an entity that a write marked stale for re-analysis, then compute and store its+ dependencies and update its dependency_status. Entities are expected to be pre-hydrated by+ [[deps.dependency-status/hydrate-for-deps]]."[entity-type entity](log/debug "Computing dependencies for" (name entity-type) (:id entity))+ (when (:stale (dependencies.db/dependency-status entity-type (:id entity)))+ (deps.findings/mark-transitive-dependents-stale! {entity-type #{(:id entity)}}))(t2/with-transaction [_](let [deps (deps.calculation/calculate-deps entity-type entity)](models.dependency/replace-dependencies! entity-type (:id entity) deps)(post-deps-cleanup! entity-type entity)(deps.dependency-status/upsert-status! entity-type (:id entity)))))+(defn- remove-deleted-entity-batch!+ "Mark the dependents of up to `batch-size` deleted entities of `entity-type` for re-analysis and drop their+ dependencies and status, returning how many were removed."+ [entity-type batch-size]+ (let [entity-ids (dependencies.db/deleted-entity-ids entity-type batch-size)]+ (when (seq entity-ids)+ (t2/with-transaction [_]+ (deps.findings/mark-transitive-dependents-stale! {entity-type entity-ids})+ (dependencies.db/delete-dependencies-from! entity-type entity-ids)+ (dependencies.db/delete-dependency-statuses! entity-type entity-ids)))+ (count entity-ids)))+(defn- backfill-entity-batch)))(defn- backfill-dependencies!- "Job to backfill dependencies for all entities.+ "Job to backfill dependencies for all entities, triggering the entity-check job for the findings it marked stale.Returns true if a full batch has been selected, nil or false otherwise."[](when (premium-features/has-feature? :dependencies)- (-> (reduce (fn [batch-size entity-type]- (if (< batch-size 1)- (reduced 0)- (let [processed (backfill-entity-batch! entity-type batch-size)]- (when (pos? processed)- (log/info "Updated" processed "entities."))- (- batch-size processed))))- (deps.settings/dependency-backfill-batch-size)- entity-types)- (< 1))))+ (let [batch-size (deps.settings/dependency-backfill-batch-size)+ remaining (reduce (fn [remaining entity-type]+ (if (< remaining 1)+ (reduced 0)+ (let [removed (remove-deleted-entity-batch! entity-type remaining)+ processed (if (< removed remaining)+ (backfill-entity-batch! entity-type (- remaining removed))+ 0)]+ (when (pos? processed)+ (log/info "Updated" processed "entities."))+ (- remaining removed processed))))+ batch-size+ entity-types)]+ (when (< remaining batch-size)+ (task.entity-check/trigger-entity-check-job!))+ (< remaining 1))))(defn- has-pending-retries?"Whether any entity is in retry backoff that this instance can actually act on.@@ -227,12 +249,3 @@(task/scheduler)(deps.task-util/job-initial-delay(deps.settings/dependency-backfill-variance-minutes))))--(events/derive! ::backfill :metabase/event)-(events/derive! :event/serdes-load ::backfill)-(events/derive! :event/set-premium-embedding-token ::backfill)--(methodical/defmethod events/publish-event! ::backfill- [_ _]- (when (premium-features/has-feature? :dependencies)- (trigger-backfill-job!)))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.cljindex 17c5a849ea2..d7b36c65ce3 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj@@ -7,11 +7,9 @@[metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding][metabase-enterprise.dependencies.settings :as deps.settings][metabase-enterprise.dependencies.task-util :as deps.task-util]- [metabase.events.core :as events][metabase.premium-features.core :as premium-features][metabase.task.core :as task]- [metabase.util.log :as log]- [methodical.core :as methodical]))+ [metabase.util.log :as log]))(set! *warn-on-reflection* true)@@ -30,16 +28,7 @@(< 1)))(defn- check-entities!- "Drain the currently-stale entities, analyzing each one.-- Staleness is propagated up front, at the moment an entity changes (see the dependencies event handlers, which mark the- whole affected subtree via [[deps.findings/mark-entity-and-transitive-dependents-stale!]]). The drain therefore works through a- fixed set that only shrinks — it never re-marks anything stale during draining — so it always terminates, even when the- dependency graph has cycles.-- As a safety net, the loop also stops as soon as a pass fails to reduce the stale count (e.g. an entity whose analysis- can't be persisted). That keeps a single unprocessable entity from spinning the loop forever; the next scheduled run- retries it."+ "Analyze stale entities in batches until none are left or a pass stops reducing their count."[](when (premium-features/has-feature? :dependencies)(loop [prev-stale nil]@@ -52,18 +41,7 @@(declare schedule-run!)(defn- reschedule-after-run!- "Schedule the next periodic run of the entity check job, unless the `:dependencies` feature is absent.-- Unlike the backfill job this one has no terminal state — it is a periodic checker, so while licensed it always queues- another run rather than stopping once the stale set is drained. Without the feature there is nothing the next run- could do, since [[check-entities!]] is gated on it, so rescheduling would just wake the instance forever. The job- restarts from the token event above, from the content-change handlers, or on the next restart.-- Scheduling deliberately fails open on an indeterminate token status: `has-feature?` cannot tell a network failure- from being unlicensed, and nothing re-fires when the check later succeeds, so reading a blip as `false` would end- the chain for good. Only a definitive `false` stops it — which is the common unlicensed case, since no token at all- answers `false`. A token that cannot be validated answers `nil` and keeps the job scheduled: if there is a token, we- keep trying. Doing the work still fails closed — see [[check-entities!]]."+ "Schedule the next run of the entity check job unless the `:dependencies` feature is definitively absent."[scheduler](if (false? (premium-features/canonically-has-feature? :dependencies))(log/debug "Not rescheduling job Dependency Entity Check: the dependencies feature is not enabled")@@ -117,14 +95,3 @@[](when (pos? (deps.settings/dependency-entity-check-batch-size))(schedule-run! (task/scheduler) 1)))--;;; Mirrors the backfill job's wiring: enabling the feature has to restart the job, because-;;; [[reschedule-after-run!]] stops the periodic chain while it is disabled and the content-change handlers that-;;; would otherwise revive it are themselves gated on the feature.-(events/derive! ::entity-check :metabase/event)-(events/derive! :event/set-premium-embedding-token ::entity-check)--(methodical/defmethod events/publish-event! ::entity-check- [_ _]- (when (premium-features/has-feature? :dependencies)- (trigger-entity-check-job!)))diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.cljindex 7da5ba2d14c..9cee0d59b82 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/sandbox.clj@@ -12,7 +12,6 @@[metabase-enterprise.sandbox.schema :as sandbox.schema][metabase.api.common :as api][metabase.audit-app.core :as audit]- [metabase.events.core :as events][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema.id :as lib.schema.id]@@ -40,7 +39,8 @@(derive :metabase/model);;; only admins can work with sandboxes(derive ::mi/read-policy.superuser)- (derive ::mi/write-policy.superuser))+ (derive ::mi/write-policy.superuser)+ (derive :hook/dependencies))(t2/deftransforms :model/Sandbox{:attribute_remappings sandbox.schema/attribute-remappings-transform})@@ -280,16 +280,8 @@(do(when (some #(contains? sandbox %) [:card_id :attribute_remappings])(sandbox.db/update-sandbox! id (u/select-keys-when sandbox :present #{:card_id :attribute_remappings})))- (let [updated-sandbox (sandbox.db/sandbox id)]- (events/publish-event! :event/sandbox-update- {:object updated-sandbox- :user-id api/*current-user-id*})- updated-sandbox))- (let [inserted-sandbox (sandbox.db/insert-sandbox! sandbox)]- (events/publish-event! :event/sandbox-create- {:object inserted-sandbox- :user-id api/*current-user-id*})- inserted-sandbox)))))+ (sandbox.db/sandbox id))+ (sandbox.db/insert-sandbox! sandbox)))))(defn- normalize-sandbox-attribute-remappings"Normalize `:attribute_remappings` on a Sandbox map, if present. `before-insert`/`before-update` hooks see thisdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj b/enterprise/backend/src/metabase_enterprise/serialization/cmd.cljindex 4ebd8d9f284..8ba5fbc41d6 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/cmd.clj@@ -10,7 +10,6 @@[metabase-enterprise.serialization.v2.storage.files :as v2.storage.files][metabase.analytics.core :as analytics][metabase.app-db.core :as mdb]- [metabase.events.core :as events][metabase.models.serialization :as serdes][metabase.plugins.core :as plugins][metabase.premium-features.core :as premium-features]@@ -53,9 +52,8 @@;;(when-not (load/compatible? path);; (log/warn "Dump was produced using a different version of Metabase. Things may break!"))(log/infof "Loading serialized Metabase files from %s" path)- (u/prog1 (serdes/with-cache- (v2.load/load-metabase! (v2.ingest/ingest-yaml path) opts))- (events/publish-event! :event/serdes-load {})))+ (serdes/with-cache+ (v2.load/load-metabase! (v2.ingest/ingest-yaml path) opts)))(mu/defn v2-load!"SerDes v2 load entry point.@@ -114,8 +112,6 @@(serdes/with-cache(-> (v2.extract/extract opts)(v2.storage/store! (v2.storage.files/file-writer path))))- ;; we could publish :event/serdes-dump to go with :event/serdes-load above, but- ;; nothing would listen to it currently(catch Exception e(reset! err e)))](analytics/track-event! :snowplow/serializationdiff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.cljindex 0ff5a078b96..fcbaa74296f 100644--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/load.clj@@ -9,6 +9,8 @@[metabase-enterprise.serialization.v2.models :as serdes.models][metabase.app-db.core :as mdb][metabase.app-db.transient-error :as transient-error]+ [metabase.dependencies.hooks :as dependencies.hooks]+ [metabase.models.interface :as mi][metabase.models.serialization :as serdes][metabase.search.core :as search][metabase.util :as u]@@ -223,8 +225,9 @@(with-retries 3 200(fn [](try- (t2/with-transaction [_tx]- (serdes/load-one! ingested local-or-nil))+ (binding [mi/*deserializing?* true]+ (t2/with-transaction [_tx]+ (serdes/load-one! ingested local-or-nil)))(catch Throwable e(serdes/reset-import-cache!)(throw e)))))@@ -285,17 +288,20 @@:files file-names}(first ingest-errors)))))(log/infof "Starting deserialization, total %s documents" (count contents))- (reduce (fn [ctx item]- (try- (load-one! ctx item)- (catch Exception e- (when-not continue-on-error- (throw e))- ;; eschew big and scary stacktrace- (log/warnf (u/strip-error e "Skipping deserialization error"))- (update ctx :errors conj e))))- ctx- contents))+ (try+ (reduce (fn [ctx item]+ (try+ (load-one! ctx item)+ (catch Exception e+ (when-not continue-on-error+ (throw e))+ ;; eschew big and scary stacktrace+ (log/warnf (u/strip-error e "Skipping deserialization error"))+ (update ctx :errors conj e))))+ ctx+ contents)+ (finally+ (dependencies.hooks/after-serdes-load))))(when reindex?;; Reindex after all entities are loaded. Individual entity commits may have produced stale;; search index entries; this ensures the index reflects the final state.diff --git a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.cljindex eeaaabe64d1..0e742df9456 100644--- a/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj+++ b/enterprise/backend/test/metabase_enterprise/data_apps/generate/resources_test.clj@@ -514,7 +514,7 @@:creator_id "[email protected]":created_at string?:card_schema pos-int?- :visualization_settings {:column_settings nil}+ :visualization_settings {}:parameters []:parameter_mappings []:dataset_query {:database (db-name)diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.cljindex aeb454922aa..90dd9755518 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj@@ -3,9 +3,8 @@[clojure.test :refer :all][medley.core :as m][metabase-enterprise.dependencies.api :as deps.api]- [metabase-enterprise.dependencies.async :as dependencies.async]- [metabase-enterprise.dependencies.events][metabase-enterprise.dependencies.findings :as dependencies.findings]+ [metabase-enterprise.dependencies.hooks :as deps.hooks][metabase-enterprise.dependencies.task.entity-check :as task.entity-check][metabase-enterprise.dependencies.test-util :as deps.test][metabase.collections.models.collection :as collection]@@ -23,9 +22,6 @@[metabase.util :as u][toucan2.core :as t2]))-(comment- metabase-enterprise.dependencies.events/keep-me)-(defn- card-with-query"Create a card map with the given name and dataset query."[card-name dataset-query]@@ -134,19 +130,6 @@[card-id](dependencies.findings/upsert-analysis! (t2/select-one :model/Card :id card-id)))-; dependencies.async/submit! effectively awaits all pending tasks on the executor.-; Those tasks would be executed regardless; this is just changing the timing to be-; less problematic for multiple test runs in sequence.-#_{:clj-kondo/ignore [:metabase/validate-deftest]}-(use-fixtures :each- (fn drained-dependency-async-executor-fixture [t]- (try- (t)- (finally- ;; Drain the single-threaded async executor to ensure all pending dependency- ;; work completes before model cleanup deletes cards, avoiding lock timeouts.- @(dependencies.async/submit! (fn [] nil))))))-(deftest graph-test(testing "GET /api/ee/dependencies/graph"(mt/with-premium-features #{:dependencies}@@ -466,11 +449,7 @@:name "inactive_transform_table"}}];; Simulate the transform having been run: link it to its (now inactive) target table(t2/update! :model/Transform transform-id {:target_table_id table-id})- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "PUBLIC"- :output-table "inactive_transform_table"- :transform-id transform-id}})+ (deps.hooks/after-transform-run transform-id table-id)(deps.test/synchronously-run-backfill!)(testing "GET /graph/dependents returns the inactive table as a child"(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/dependents"@@ -1198,11 +1177,7 @@;; Simulate the referenced transform having been run: link it to its target table;; so the table→transform dep is visible in the dependency graph.(t2/update! :model/Transform referenced-transform-id {:target_table_id referenced-table-id})- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "PUBLIC"- :output-table "referenced_transform_table"- :transform-id referenced-transform-id}})+ (deps.hooks/after-transform-run referenced-transform-id referenced-table-id)(deps.test/synchronously-run-backfill!)(let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=transform&query=unreftest")](is (=? {:data [{:id unreffed-transform-iddiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.cljindex b9f5efadd08..57006cb47d4 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/events_test.clj@@ -1,23 +1,21 @@(ns metabase-enterprise.dependencies.events-test(:require[clojure.test :refer [deftest is testing]]- [metabase-enterprise.dependencies.events][metabase-enterprise.dependencies.findings :as deps.findings]+ [metabase-enterprise.dependencies.hooks :as deps.hooks][metabase-enterprise.dependencies.models.analysis-finding :as models.analysis-finding][metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status][metabase-enterprise.dependencies.task.entity-check :as task.entity-check][metabase-enterprise.dependencies.test-util :as deps.test]- [metabase.api.common :as api]- [metabase.events.core :as events]+ [metabase-enterprise.serialization.test-util :as ts]+ [metabase-enterprise.serialization.v2.load :as serdes.load][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.models.serialization :as serdes][metabase.test :as mt][toucan2.core :as t2]))-(comment- metabase-enterprise.dependencies.events/keep-me)-(defn- run-with-dependencies-setup! [thunk](mt/with-test-user :rasta(mt/with-premium-features #{:dependencies}@@ -32,6 +30,12 @@(is (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true)(str "Expected " (name entity-type) " " entity-id " to be marked stale")))+(defn- assert-pending+ "Assert that the given entity awaits the backfill job, having no dependency_status row or a stale one."+ [entity-type entity-id]+ (is (not (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale false))+ (str "Expected " (name entity-type) " " entity-id " to be pending")))+(deftest dashboard-update-sets-correct-dependencies(run-with-dependencies-setup!(fn [mp]@@ -45,17 +49,16 @@:model/Card {param-source-card-id :id} {:dataset_query (-> (lib/query mp products)(lib/aggregate (lib/count))(lib/breakout category-field))}- :model/Dashboard {dashboard-id :id :as dashboard} {:parameters [{:id "category-param"- :type "category"- :values_source_type "card"- :values_source_config {:card_id param-source-card-id}}]}+ :model/Dashboard {dashboard-id :id} {:parameters [{:id "category-param"+ :type "category"+ :values_source_type "card"+ :values_source_config {:card_id param-source-card-id}}]}:model/DashboardCard {basic-dashcard-id :id} {:dashboard_id dashboard-id:card_id basic-card-id}:model/DashboardCardSeries _ {:dashboardcard_id basic-dashcard-id:card_id series-card-id:position 0}]- (events/publish-event! :event/dashboard-create {:object dashboard :user-id api/*current-user-id*})- (assert-stale :dashboard dashboard-id)+ (assert-pending :dashboard dashboard-id)(deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :dashboard:from_entity_id dashboard-id@@ -111,46 +114,44 @@:visualization_settings {:column_settings {"[\"name\",\"CATEGORY\"]" {:click_behavior {:type "link":linkType "dashboard":targetId column-click-target-dashboard-id}}}}}]- (let [updated-dashboard (t2/select-one :model/Dashboard :id dashboard-id)]- (events/publish-event! :event/dashboard-update {:object updated-dashboard :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (=? #{{:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :card- :to_entity_id scalar-card-id}- {:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :card- :to_entity_id scalar-click-target-card-id}- {:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :card- :to_entity_id scalar-card-2-id}- {:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :dashboard- :to_entity_id scalar-click-target-dashboard-id}- {:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :card- :to_entity_id table-card-id}- {:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :card- :to_entity_id column-click-target-card-id}- {:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :card- :to_entity_id table-card-2-id}- {:from_entity_type :dashboard- :from_entity_id dashboard-id- :to_entity_type :dashboard- :to_entity_id column-click-target-dashboard-id}}- (into #{} (map #(dissoc % :id)- (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))- (t2/delete! :model/Dashboard dashboard-id)- (events/publish-event! :event/dashboard-delete {:object updated-dashboard :user-id api/*current-user-id*})- (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard))))))))))+ (deps.test/synchronously-run-backfill!)+ (is (=? #{{:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :card+ :to_entity_id scalar-card-id}+ {:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :card+ :to_entity_id scalar-click-target-card-id}+ {:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :card+ :to_entity_id scalar-card-2-id}+ {:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :dashboard+ :to_entity_id scalar-click-target-dashboard-id}+ {:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :card+ :to_entity_id table-card-id}+ {:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :card+ :to_entity_id column-click-target-card-id}+ {:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :card+ :to_entity_id table-card-2-id}+ {:from_entity_type :dashboard+ :from_entity_id dashboard-id+ :to_entity_type :dashboard+ :to_entity_id column-click-target-dashboard-id}}+ (into #{} (map #(dissoc % :id)+ (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))+ (t2/delete! :model/Dashboard dashboard-id)+ (deps.test/synchronously-run-backfill!)+ (is (empty? (t2/select :model/Dependency :from_entity_id dashboard-id :from_entity_type :dashboard)))))))))(deftest document-update-sets-correct-dependencies(run-with-dependencies-setup!@@ -169,8 +170,7 @@"model" "card"}}]}{:type "cardEmbed":attrs {"id" embedded-card-id}}]}}]- (events/publish-event! :event/document-create {:object document :user-id api/*current-user-id*})- (assert-stale :document document-id)+ (assert-pending :document document-id)(deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :document:from_entity_id document-id@@ -191,7 +191,6 @@:attrs {"entityId" products-id"model" "table"}}]}]})](t2/update! :model/Document document-id updated-doc)- (events/publish-event! :event/document-update {:object updated-doc :user-id api/*current-user-id*})(deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :document:from_entity_id document-id@@ -204,7 +203,7 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document)))))(t2/delete! :model/Document document-id)- (events/publish-event! :event/document-delete {:object document :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (empty? (t2/select :model/Dependency :from_entity_id document-id :from_entity_type :document))))))))))(deftest sandbox-update-sets-correct-dependencies@@ -217,8 +216,7 @@:model/Sandbox {sandbox-id :id :as sandbox} {:group_id group-id:table_id (mt/id :products):card_id card1-id}]- (events/publish-event! :event/sandbox-create {:object sandbox :user-id api/*current-user-id*})- (assert-stale :sandbox sandbox-id)+ (assert-pending :sandbox sandbox-id)(deps.test/synchronously-run-backfill!)(is (=? #{{:from_entity_type :sandbox:from_entity_id sandbox-id@@ -227,31 +225,33 @@(into #{} (map #(dissoc % :id)(t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))(t2/update! :model/Sandbox sandbox-id (assoc sandbox :card_id card2-id))- (let [updated-sandbox (t2/select-one :model/Sandbox :id sandbox-id)]- (events/publish-event! :event/sandbox-update {:object updated-sandbox :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (=? #{{:from_entity_type :sandbox- :from_entity_id sandbox-id- :to_entity_type :card- :to_entity_id card2-id}}- (into #{} (map #(dissoc % :id)- (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))- (t2/delete! :model/Sandbox sandbox-id)- (events/publish-event! :event/sandbox-delete {:object sandbox :user-id api/*current-user-id*})- (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))))))+ (deps.test/synchronously-run-backfill!)+ (is (=? #{{:from_entity_type :sandbox+ :from_entity_id sandbox-id+ :to_entity_type :card+ :to_entity_id card2-id}}+ (into #{} (map #(dissoc % :id)+ (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox)))))+ (t2/delete! :model/Sandbox sandbox-id)+ (deps.test/synchronously-run-backfill!)+ (is (empty? (t2/select :model/Dependency :from_entity_id sandbox-id :from_entity_type :sandbox))))))))-(deftest card-marks-stale-on-create-update-test- (testing "Card create/update events mark the entity stale in dependency_status"+(deftest card-marks-pending-on-create-and-stale-on-update-test+ (testing "A created card is pending for the backfill job, and changing its query marks it stale in dependency_status"(run-with-dependencies-setup!(fn [mp]- (let [products (lib.metadata/table mp (mt/id :products))]- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})- (assert-stale :card card-id)+ (let [products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]+ (assert-pending :card card-id)(deps.test/synchronously-run-backfill!)(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id (mt/id :products)))- (events/publish-event! :event/card-update {:object card :previous-object card :user-id api/*current-user-id*})+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false))+ (testing "a change to a column the dependencies do not come from leaves it up to date"+ (t2/update! :model/Card card-id {:name "renamed"})+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale false)))+ (t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)})(assert-stale :card card-id)))))))(deftest metric-dimensions-update-marks-stale-and-includes-mapped-table-test@@ -265,18 +265,11 @@;; uncurated, so the after-insert dimension auto-sync would seed dimensions from the query and;; overwrite mappings passed to with-temp. A plain update sticks — the after-update hook only;; re-syncs when `:dataset_query` changes.- ;; The dimension endpoints announce their writes as card updates; see- ;; [[metabase.metrics.api/notify-dimensions-changed!]].- (let [before (t2/select-one :model/Card :id card-id)]- (t2/update! :model/Card card-id- {:dimension_mappings [{:type :table- :dimension-id "550e8400-e29b-41d4-a716-446655440000"- :table-id (mt/id :categories)- :target [:field {} (mt/id :categories :name)]}]})- (events/publish-event! :event/card-update- {:object (t2/select-one :model/Card :id card-id)- :previous-object before- :user-id api/*current-user-id*}))+ (t2/update! :model/Card card-id+ {:dimension_mappings [{:type :table+ :dimension-id "550e8400-e29b-41d4-a716-446655440000"+ :table-id (mt/id :categories)+ :target [:field {} (mt/id :categories :name)]}]})(assert-stale :card card-id)(deps.test/synchronously-run-backfill!)(is (t2/exists? :model/Dependency :from_entity_type :card :from_entity_id card-id@@ -286,40 +279,41 @@:to_entity_type :table :to_entity_id (mt/id :venues))"the query's own table remains a dependency")))))))-(deftest snippet-marks-stale-on-create-update-test- (testing "Snippet create/update events mark the entity stale in dependency_status"+(deftest snippet-marks-pending-on-create-and-stale-on-update-test+ (testing "A created snippet is pending for the backfill job, and changing its content marks it stale in dependency_status"(run-with-dependencies-setup!(fn [_]- (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id :as snippet} {:name "test snippet"- :content "SELECT 1"}]- (events/publish-event! :event/snippet-create {:object snippet :user-id api/*current-user-id*})- (assert-stale :snippet snippet-id)- (events/publish-event! :event/snippet-update {:object snippet :user-id api/*current-user-id*})+ (mt/with-temp [:model/NativeQuerySnippet {snippet-id :id} {:name "test snippet"+ :content "SELECT 1"}]+ (assert-pending :snippet snippet-id)+ (deps.test/synchronously-run-backfill!)+ (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id snippet-id :stale false))+ (t2/update! :model/NativeQuerySnippet snippet-id {:content "SELECT 2"})(assert-stale :snippet snippet-id))))))-(deftest mark-stale-failure-does-not-propagate-test- (testing "When mark-stale! throws, the event handler catches the error and the API call is not affected"+(deftest mark-stale-failure-fails-the-write-test+ (testing "When mark-stale! throws, the update fails and the card keeps its query"(run-with-dependencies-setup!(fn [mp]- (let [products (lib.metadata/table mp (mt/id :products))]- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (lib/query mp products)}]+ (let [products (lib.metadata/table mp (mt/id :products))+ orders (lib.metadata/table mp (mt/id :orders))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}]+ (deps.test/synchronously-run-backfill!)(mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!- (fn [_ _] (throw (ex-info "Simulated DB failure" {})))]- ;; Should not throw — the error is caught and logged- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*}))- ;; Entity should NOT be stale (mark-stale! failed)- (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id card-id :stale true))- "Entity should not be marked stale when mark-stale! fails")))))))+ (fn [_] (throw (ex-info "Simulated DB failure" {})))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Simulated DB failure"+ (t2/update! :model/Card card-id {:dataset_query (lib/query mp orders)}))))+ (is (=? {:stages [{:source-table (mt/id :products)}]}+ (t2/select-one-fn :dataset_query :model/Card card-id)))))))))-(deftest ^:synchronized native-transform-updates-dependencies-test- (testing "native transform update events trigger dependency calculations"+(deftest ^:synchronized native-transform-create-sets-dependencies-test+ (testing "a created native transform is picked up by the dependency calculation"(run-with-dependencies-setup!(fn [mp](let [source {:query (lib/native-query mp "select * from orders"):type :query}]- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (assert-stale :transform transform-id)+ (mt/with-temp [:model/Transform {transform-id :id} {:source source}]+ (assert-pending :transform transform-id)(deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,@@ -327,15 +321,14 @@:to_entity_id (mt/id :orders)}](t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))-(deftest ^:synchronized mbql-transform-updates-dependencies-test- (testing "mbql transform update events trigger dependency calculations"+(deftest ^:synchronized mbql-transform-create-sets-dependencies-test+ (testing "a created mbql transform is picked up by the dependency calculation"(run-with-dependencies-setup!(fn [mp](let [source {:query (lib/query mp (lib.metadata/table mp (mt/id :orders))):type :query}]- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (assert-stale :transform transform-id)+ (mt/with-temp [:model/Transform {transform-id :id} {:source source}]+ (assert-pending :transform transform-id)(deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,@@ -343,8 +336,8 @@:to_entity_id (mt/id :orders)}](t2/select :model/Dependency :from_entity_id transform-id :from_entity_type :transform)))))))))-(deftest ^:synchronized python-transform-updates-dependencies-test- (testing "python transform update events trigger dependency calculations"+(deftest ^:synchronized python-transform-create-sets-dependencies-test+ (testing "a created python transform is picked up by the dependency calculation"(run-with-dependencies-setup!(fn [_](let [source {:type :python,@@ -356,9 +349,8 @@:table_id (mt/id :orders)}],:body"import pandas as pd\n\ndef transform(orders):\n return orders"}]- (mt/with-temp [:model/Transform {transform-id :id :as transform} {:source source}]- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})- (assert-stale :transform transform-id)+ (mt/with-temp [:model/Transform {transform-id :id} {:source source}]+ (assert-pending :transform transform-id)(deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :transform,:from_entity_id transform-id,@@ -390,10 +382,12 @@(t2/delete! :model/Database db-id)(is (nil? (t2/select-one :model/Database db-id))"the Database row is actually gone")+ (assert-stale :transform transform-id)+ (deps.test/synchronously-run-backfill!)(is (true? (t2/select-one-fn :stale :model/AnalysisFinding:analyzed_entity_type :transform:analyzed_entity_id transform-id))- "after db-delete the transform's finding is marked stale")+ "after db-delete the backfill marks the transform's finding stale")(testing "and re-running the entity-check job flips :result false so the transform shows on broken-diagnostics"(#'task.entity-check/check-entities!)(let [{:keys [stale result]} (t2/select-one :model/AnalysisFinding@@ -416,12 +410,7 @@(let [target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}](mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}:model/Transform {transform-id :id} {:target target}]- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})+ (deps.hooks/after-transform-run transform-id table-id)(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,@@ -429,7 +418,7 @@(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))(deftest ^:synchronized python-transform-update-handles-downstream-dependencies-test- (testing "python transform update events handles downstream dependencies"+ (testing "python transform updates handle downstream dependencies"(run-with-dependencies-setup!(fn [_](let [source {:type :python,@@ -444,24 +433,18 @@target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}](mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}- :model/Transform {transform-id :id :as transform} {:target target- :source source}]+ :model/Transform {transform-id :id} {:target target+ :source source}](testing "initial run"- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})+ (deps.hooks/after-transform-run transform-id table-id)(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,:to_entity_id transform-id}](t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "keeping target"- (events/publish-event! :event/update-transform- {:object transform- :user-id api/*current-user-id*})+ (t2/update! :model/Transform transform-id {:source (update source :body str "\n")})+ (assert-stale :transform transform-id)(deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :table:from_entity_id table-id,@@ -470,16 +453,12 @@(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "changing target update"(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})- (let [updated (t2/select-one :model/Transform :id transform-id)]- (events/publish-event! :event/update-transform- {:object updated- :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))+ (deps.test/synchronously-run-backfill!)+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))(deftest ^:synchronized query-transform-update-handles-downstream-dependencies-test- (testing "query transform update events handles downstream dependencies"+ (testing "query transform updates handle downstream dependencies"(run-with-dependencies-setup!(fn [mp](let [source {:query (lib/native-query mp "select * from orders")@@ -487,23 +466,17 @@target {:type "table", :schema "Other", :name "test_table", :database (mt/id)}](mt/with-temp [:model/Table {table-id :id} {:schema "Other", :db_id (mt/id), :name "test_table"}:model/Table {} {:schema "Other", :db_id (mt/id), :name "test_table2"}- :model/Transform {transform-id :id :as transform} {:target target :source source}]+ :model/Transform {transform-id :id} {:target target :source source}](testing "initial run"- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "Other"- :output-table :test_table- :transform-id transform-id}- :user-id api/*current-user-id*})+ (deps.hooks/after-transform-run transform-id table-id)(is (=? [{:from_entity_type :table:from_entity_id table-id,:to_entity_type :transform,:to_entity_id transform-id}](t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "keeping target"- (events/publish-event! :event/update-transform- {:object transform- :user-id api/*current-user-id*})+ (t2/update! :model/Transform transform-id {:source (assoc source :query (lib/native-query mp "select * from orders limit 10"))})+ (assert-stale :transform transform-id)(deps.test/synchronously-run-backfill!)(is (=? [{:from_entity_type :table:from_entity_id table-id,@@ -512,13 +485,9 @@(t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))(testing "changing target update"(t2/update! :model/Transform transform-id {:target (assoc target :name "test_table2")})- (let [updated (t2/select-one :model/Transform :id transform-id)]- (events/publish-event! :event/update-transform- {:object updated- :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (empty?- (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform)))))))))))+ (deps.test/synchronously-run-backfill!)+ (is (empty?+ (t2/select :model/Dependency :to_entity_id transform-id :to_entity_type :transform))))))))))(deftest segment-update-sets-correct-dependencies(run-with-dependencies-setup!@@ -526,11 +495,10 @@(let [products-id (mt/id :products)price-field-id (mt/id :products :price)category-field-id (mt/id :products :category)]- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id- :definition {:filter [:> [:field price-field-id nil] 50]}}]+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 50]}}](testing "creating a segment creates dependency to its table"- (events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})- (assert-stale :segment segment-id)+ (assert-pending :segment segment-id)(deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :segment:from_entity_id segment-id@@ -540,18 +508,17 @@(t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))(testing "updating segment definition recalculates dependencies"(t2/update! :model/Segment segment-id {:definition {:filter [:= [:field category-field-id nil] "Widget"]}})- (let [updated-segment (t2/select-one :model/Segment :id segment-id)]- (events/publish-event! :event/segment-update {:object updated-segment :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (= #{{:from_entity_type :segment- :from_entity_id segment-id- :to_entity_type :table- :to_entity_id products-id}}- (into #{} (map #(dissoc % :id)- (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))+ (assert-stale :segment segment-id)+ (deps.test/synchronously-run-backfill!)+ (is (= #{{:from_entity_type :segment+ :from_entity_id segment-id+ :to_entity_type :table+ :to_entity_id products-id}}+ (into #{} (map #(dissoc % :id)+ (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment))))))(testing "deleting segment removes all dependencies"(t2/delete! :model/Segment segment-id)- (events/publish-event! :event/segment-delete {:object segment :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (empty? (t2/select :model/Dependency :from_entity_id segment-id :from_entity_type :segment)))))))))(deftest card-with-segment-dependencies@@ -559,16 +526,14 @@(fn [_](let [products-id (mt/id :products)price-field-id (mt/id :products :price)]- (mt/with-temp [:model/Segment {segment-id :id :as segment} {:table_id products-id- :definition {:filter [:> [:field price-field-id nil] 50]}}]+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 50]}}](testing "creating a card using a segment creates dependencies to both segment and table"- (events/publish-event! :event/segment-create {:object segment :user-id api/*current-user-id*})(deps.test/synchronously-run-backfill!)- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query {:database (mt/id)- :type :query- :query {:source-table products-id- :filter [:segment segment-id]}}}]- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query {:database (mt/id)+ :type :query+ :query {:source-table products-id+ :filter [:segment segment-id]}}}](deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :card:from_entity_id card-id@@ -589,12 +554,11 @@orders (lib.metadata/table mp orders-id)quantity (lib.metadata/field mp (mt/id :orders :quantity))subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]- (mt/with-temp [:model/Measure {measure-id :id :as measure} {:table_id orders-id- :definition (-> (lib/query mp orders)- (lib/aggregate (lib/sum quantity)))}]+ (mt/with-temp [:model/Measure {measure-id :id} {:table_id orders-id+ :definition (-> (lib/query mp orders)+ (lib/aggregate (lib/sum quantity)))}](testing "creating a measure creates dependency to its table"- (events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})- (assert-stale :measure measure-id)+ (assert-pending :measure measure-id)(deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :measure:from_entity_id measure-id@@ -605,18 +569,17 @@(testing "updating measure definition recalculates dependencies"(t2/update! :model/Measure measure-id {:definition (-> (lib/query mp orders)(lib/aggregate (lib/sum subtotal)))})- (let [updated-measure (t2/select-one :model/Measure :id measure-id)]- (events/publish-event! :event/measure-update {:object updated-measure :user-id api/*current-user-id*})- (deps.test/synchronously-run-backfill!)- (is (= #{{:from_entity_type :measure- :from_entity_id measure-id- :to_entity_type :table- :to_entity_id orders-id}}- (into #{} (map #(dissoc % :id)- (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure)))))))+ (assert-stale :measure measure-id)+ (deps.test/synchronously-run-backfill!)+ (is (= #{{:from_entity_type :measure+ :from_entity_id measure-id+ :to_entity_type :table+ :to_entity_id orders-id}}+ (into #{} (map #(dissoc % :id)+ (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))(testing "deleting measure removes all dependencies"(t2/delete! :model/Measure measure-id)- (events/publish-event! :event/measure-delete {:object measure :user-id api/*current-user-id*})+ (deps.test/synchronously-run-backfill!)(is (empty? (t2/select :model/Dependency :from_entity_id measure-id :from_entity_type :measure))))))))))(deftest measure-with-measure-dependencies@@ -626,17 +589,15 @@orders (lib.metadata/table mp orders-id)quantity (lib.metadata/field mp (mt/id :orders :quantity))subtotal (lib.metadata/field mp (mt/id :orders :subtotal))]- (mt/with-temp [:model/Measure {measure1-id :id :as measure1} {:table_id orders-id- :definition (-> (lib/query mp orders)- (lib/aggregate (lib/sum quantity)))}]+ (mt/with-temp [:model/Measure {measure1-id :id} {:table_id orders-id+ :definition (-> (lib/query mp orders)+ (lib/aggregate (lib/sum quantity)))}](let [mp' (mt/metadata-provider)]- (mt/with-temp [:model/Measure {measure2-id :id :as measure2} {:table_id orders-id- :definition (-> (lib/query mp' orders)- (lib/aggregate (lib/+ (lib.metadata/measure mp' measure1-id)- (lib/sum subtotal))))}]+ (mt/with-temp [:model/Measure {measure2-id :id} {:table_id orders-id+ :definition (-> (lib/query mp' orders)+ (lib/aggregate (lib/+ (lib.metadata/measure mp' measure1-id)+ (lib/sum subtotal))))}](testing "creating a measure that references another measure creates dependencies to both"- (events/publish-event! :event/measure-create {:object measure1 :user-id api/*current-user-id*})- (events/publish-event! :event/measure-create {:object measure2 :user-id api/*current-user-id*})(deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :measure:from_entity_id measure2-id@@ -655,17 +616,15 @@(let [orders-id (mt/id :orders)orders (lib.metadata/table mp orders-id)quantity (lib.metadata/field mp (mt/id :orders :quantity))]- (mt/with-temp [:model/Measure {measure-id :id :as measure} {:table_id orders-id- :definition (-> (lib/query mp orders)- (lib/aggregate (lib/sum quantity)))}]- (events/publish-event! :event/measure-create {:object measure :user-id api/*current-user-id*})+ (mt/with-temp [:model/Measure {measure-id :id} {:table_id orders-id+ :definition (-> (lib/query mp orders)+ (lib/aggregate (lib/sum quantity)))}](deps.test/synchronously-run-backfill!)(testing "creating a card using a measure creates dependencies to both measure and table"(let [mp' (mt/metadata-provider)query (-> (lib/query mp' orders)(lib/aggregate (lib.metadata/measure mp' measure-id)))]- (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query query}]- (events/publish-event! :event/card-create {:object card :user-id api/*current-user-id*})+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query}](deps.test/synchronously-run-backfill!)(is (= #{{:from_entity_type :card:from_entity_id card-id@@ -705,13 +664,13 @@(testing "Card updates should mark the card stale and trigger the background job"(let [products-id (mt/id :products)products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {parent-card-id :id :as parent-card} {:dataset_query (lib/query mp products)}+ (mt/with-temp [:model/Card {parent-card-id :id} {:dataset_query (lib/query mp products)}:model/Card {child-card-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-card-id))}:model/Dependency _ {:from_entity_type :card:from_entity_id child-card-id:to_entity_type :card:to_entity_id parent-card-id}]- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp (lib.metadata/table mp (mt/id :orders)))});; No analysis record should exist yet - mark-entity-stale! is a no-op for never-analyzed entities(assert-has-analyses{:card {parent-card-id nil}})@@ -739,17 +698,23 @@(deps.findings/upsert-analysis! parent-card)(deps.findings/upsert-analysis! child-card)(deps.findings/upsert-analysis! other-card)+ (deps.dependency-status/upsert-status! :card child-card-id)+ (deps.dependency-status/upsert-status! :card other-card-id)(testing "Checking that the initial analyses exist"(assert-has-analyses{:card {parent-card-id old-versionchild-card-id old-versionother-card-id old-version}}))- (testing "card update only marks direct dependents stale - not its own analysis"- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ (testing "changing the card query marks its finding and its dependents' stale, not unrelated ones"+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)})+ (deps.test/synchronously-run-backfill!)(assert-has-analyses{:card {parent-card-id old-versionchild-card-id -1- other-card-id old-version}}))+ other-card-id old-version}})+ (is (= {parent-card-id true child-card-id true other-card-id false}+ (t2/select-fn->fn :analyzed_entity_id :stale :model/AnalysisFinding :analyzed_entity_type :card+ :analyzed_entity_id [:in [parent-card-id child-card-id other-card-id]]))))(testing "If the version changes, the job picks up only the stale cards"(binding [models.analysis-finding/*current-analysis-finding-version* (inc old-version)](assert-has-analyses@@ -769,8 +734,9 @@:to_entity_type :card :to_entity_id parent-id}](deps.findings/upsert-analysis! parent)(deps.findings/upsert-analysis! child)- ;; Event marks entity stale in analysis_finding- (events/publish-event! :event/card-update {:object parent :previous-object parent :user-id api/*current-user-id*})+ (deps.dependency-status/upsert-status! :card child-id)+ (t2/update! :model/Card parent-id {:dataset_query (lib/native-query mp "select * from orders")})+ (deps.test/synchronously-run-backfill!)(testing "Parent card should be marked stale"(is (true? (t2/select-one-fn :stale :model/AnalysisFinding:analyzed_entity_type :card@@ -811,8 +777,7 @@(assert-has-analyses{:card {parent-card-id old-version child-card-id old-version}:transform {transform-id old-version}})- ;; Event marks parent stale in analysis_finding, which triggers entity-check- (events/publish-event! :event/card-update {:object parent-card :previous-object parent-card :user-id api/*current-user-id*})+ (t2/update! :model/Card parent-card-id {:dataset_query (lib/query mp orders)});; Run entity-check job to drain the stale entities(#'task.entity-check/check-entities!)(testing "All entities should be re-analyzed after the drain"@@ -828,14 +793,15 @@(fn [mp](testing "Transform update does not error when no analyses exist yet"(let [products (lib.metadata/table mp (mt/id :products))]- (mt/with-temp [:model/Transform transform {:source {:type :query- :query (lib/query mp products)}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}]- ;; Should not throw even when no analysis findings exist- (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})))))))))+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query+ :query (lib/query mp products)}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ (is (= 1 (t2/update! :model/Transform transform-id+ {:source {:type :query+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})))))))))(deftest ^:synchronized transform-update-updates-analyses-test(run-with-dependencies-setup!@@ -851,7 +817,9 @@:type :table}}](deps.findings/upsert-analysis! transform)(assert-has-analyses {:transform {transform-id old-version}})- (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*})+ (t2/update! :model/Transform transform-id {:source {:type :query+ :query (lib/query mp (lib.metadata/table mp (mt/id :orders)))}})+ (deps.test/synchronously-run-backfill!)(testing "Transform should be marked stale"(is (true? (t2/select-one-fn :stale :model/AnalysisFinding:analyzed_entity_type :transform@@ -866,14 +834,15 @@(run-with-dependencies-setup!(fn [mp](testing "Native transform updates do not error"- (mt/with-temp [:model/Transform transform {:source {:type :query- :query (lib/native-query mp "select * from products")}- :name "transform_sample"- :target {:schema "public"- :name "sample"- :type :table}}]- ;; Should not throw even for native transforms- (is (some? (events/publish-event! :event/update-transform {:object transform :user-id api/*current-user-id*}))))))))+ (mt/with-temp [:model/Transform {transform-id :id} {:source {:type :query+ :query (lib/native-query mp "select * from products")}+ :name "transform_sample"+ :target {:schema "public"+ :name "sample"+ :type :table}}]+ (is (= 1 (t2/update! :model/Transform transform-id+ {:source {:type :query+ :query (lib/native-query mp "select * from orders")}}))))))))(deftest ^:synchronized transform-run-works-with-no-analyses-test(run-with-dependencies-setup!@@ -886,12 +855,7 @@:target {:schema "public" :name "sample" :type :table}}:model/Dependency _ {:from_entity_type :card :from_entity_id card-id:to_entity_type :transform :to_entity_id transform-id}]- (is (some? (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "public"- :output-table "sample"- :transform-id transform-id}- :user-id api/*current-user-id*})))))))))+ (is (nil? (deps.hooks/after-transform-run transform-id nil)))))))))(deftest ^:synchronized transform-run-marks-dependents-stale-test(run-with-dependencies-setup!@@ -909,12 +873,7 @@(deps.findings/upsert-analysis! card)(deps.findings/upsert-analysis! transform)(deps.findings/upsert-analysis! other-card)- (events/publish-event! :event/transform-run-complete- {:object {:db-id (mt/id)- :output-schema "public"- :output-table "sample"- :transform-id transform-id}- :user-id api/*current-user-id*})+ (deps.hooks/after-transform-run transform-id nil)(testing "Dependent card should be marked stale"(is (true? (t2/select-one-fn :stale :model/AnalysisFinding:analyzed_entity_type :card :analyzed_entity_id card-id))))@@ -931,10 +890,10 @@(testing "Segment update does not error when no analyses exist yet"(let [products-id (mt/id :products)price-field-id (mt/id :products :price)]- (mt/with-temp [:model/Segment segment {:table_id products-id- :definition {:filter [:> [:field price-field-id nil] 50]}}]- ;; Should not throw even when no analysis findings exist- (is (some? (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})))))))))+ (mt/with-temp [:model/Segment {segment-id :id} {:table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 50]}}]+ (is (= 1 (t2/update! :model/Segment segment-id+ {:definition {:filter [:< [:field price-field-id nil] 50]}})))))))))(deftest ^:synchronized segment-update-updates-analyses-test(run-with-dependencies-setup!@@ -951,8 +910,10 @@:to_entity_type :segment :to_entity_id segment-id}](deps.findings/upsert-analysis! segment)(deps.findings/upsert-analysis! card)+ (deps.dependency-status/upsert-status! :card card-id)(assert-has-analyses {:card {card-id old-version} :segment {segment-id old-version}})- (events/publish-event! :event/segment-update {:object segment :user-id api/*current-user-id*})+ (t2/update! :model/Segment segment-id {:definition {:filter [:< [:field price-field-id nil] 50]}})+ (deps.test/synchronously-run-backfill!)(testing "Segment should be marked stale"(is (true? (t2/select-one-fn :stale :model/AnalysisFinding:analyzed_entity_type :segment :analyzed_entity_id segment-id))))@@ -978,14 +939,14 @@(deps.findings/upsert-analysis! card)(deps.findings/upsert-analysis! other-card)(assert-has-analyses {:card {card-id old-version other-card-id old-version}})- (let [table (t2/select-one :model/Table :id products-id)]- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})- (testing "Card depending on updated table should be marked stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id card-id))))- (testing "Card not depending on updated table should not be stale"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))+ (deps.test/do-with-immediate-hook-writes+ #(mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}]+ (testing "Card depending on updated table should be marked stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id card-id))))+ (testing "Card not depending on updated table should not be stale"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))(deftest ^:synchronized table-metadata-update-works-with-no-analyses-test(run-with-dependencies-setup!@@ -996,8 +957,7 @@(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}:model/Dependency _ {:from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id products-id}]- (let [table (t2/select-one :model/Table :id products-id)]- (events/publish-event! :event/table-update {:object table :user-id api/*current-user-id*})+ (mt/with-temp [:model/TableUserSettings _ {:table_id products-id :display_name "Products renamed"}](assert-has-analyses {:card {card-id nil}}))))))))(deftest ^:synchronized field-metadata-update-triggers-dependent-analysis-test@@ -1015,14 +975,14 @@(deps.findings/upsert-analysis! card)(deps.findings/upsert-analysis! other-card)(assert-has-analyses {:card {card-id old-version other-card-id old-version}})- (let [field (t2/select-one :model/Field :id (mt/id :products :category))]- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})- (testing "Card depending on the table whose field was updated should be stale"- (is (true? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id card-id))))- (testing "Card not depending on that table should not be stale"- (is (false? (t2/select-one-fn :stale :model/AnalysisFinding- :analyzed_entity_type :card :analyzed_entity_id other-card-id)))))))))))+ (deps.test/do-with-immediate-hook-writes+ #(mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}]+ (testing "Card depending on the table whose field was updated should be stale"+ (is (true? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id card-id))))+ (testing "Card not depending on that table should not be stale"+ (is (false? (t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id other-card-id))))))))))))(deftest ^:synchronized field-metadata-update-works-with-no-analyses-test(run-with-dependencies-setup!@@ -1033,6 +993,32 @@(mt/with-temp [:model/Card {card-id :id} {:dataset_query (lib/query mp products)}:model/Dependency _ {:from_entity_type :card :from_entity_id card-id:to_entity_type :table :to_entity_id products-id}]- (let [field (t2/select-one :model/Field :id (mt/id :products :category))]- (events/publish-event! :event/field-update {:object field :user-id api/*current-user-id*})+ (mt/with-temp [:model/FieldUserSettings _ {:field_id (mt/id :products :category) :display_name "Category renamed"}](assert-has-analyses {:card {card-id nil}}))))))))++(deftest serdes-load-recalculates-changed-entities-test+ (testing "a serdes load recalculates the dependencies of the entities it changes, and only those"+ (run-with-dependencies-setup!+ (fn [mp]+ (let [card-deps #(t2/select-fn-set (juxt :to_entity_type :to_entity_id)+ :model/Dependency :from_entity_type :card :from_entity_id %)+ query #(lib/query mp (lib.metadata/table mp (mt/id %)))+ processed #(t2/exists? :model/DependencyStatus :entity_type :card :entity_id % :stale false)+ load! #(serdes/with-cache+ (serdes.load/load-metabase! (ts/ingestion-in-memory %) :reindex? false))]+ (mt/with-temp [:model/Card {card-id :id :as card} {:dataset_query (query :products)}+ :model/Card {other-id :id} {:dataset_query (query :products)}]+ (let [exported (serdes/extract-one "Card" {} card)]+ (t2/update! :model/Card card-id {:dataset_query (query :orders)})+ (deps.dependency-status/mark-stale! {:card #{card-id other-id}})+ (deps.test/synchronously-run-backfill!)+ (is (= #{[:table (mt/id :orders)]} (card-deps card-id)))+ (load! [exported])+ (testing "only the changed card is marked stale"+ (assert-stale :card card-id)+ (is (processed other-id)))+ (deps.test/synchronously-run-backfill!)+ (is (= #{[:table (mt/id :products)]} (card-deps card-id)))+ (testing "reloading it unchanged marks nothing"+ (load! [exported])+ (is (processed card-id))))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.cljindex c8623687cfe..9dee6dc38fb 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/findings_test.clj@@ -330,7 +330,7 @@(let [cid (:id card)];; seed a clean finding, mark it stale, then make its database unresolvable(models.analysis-finding/upsert-analysis! :card cid true [])- (models.analysis-finding/mark-stale! :card [cid])+ (models.analysis-finding/mark-stale! {:card #{cid}})(with-redefs-fn {#'deps.findings/instance-db-id (constantly nil)}(fn [](lib-be/with-metadata-provider-cache@@ -363,7 +363,7 @@:analyzed_entity_type :card :analyzed_entity_id cid))"phase 1: stale flag cleared");; phase 2: database resolvable again; re-trigger analysis (as an event/job would)- (models.analysis-finding/mark-stale! :card [cid])+ (models.analysis-finding/mark-stale! {:card #{cid}})(lib-be/with-metadata-provider-cache(deps.findings/upsert-analysis! (t2/instance :model/Card card)))(is (empty? (models.analysis-finding-error/errors-for-entity :card cid))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.cljnew file mode 100644index 00000000000..d3e8dec01e6--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/hooks_test.clj@@ -0,0 +1,190 @@+(ns metabase-enterprise.dependencies.hooks-test+ (:require+ [clojure.test :refer :all]+ [metabase-enterprise.dependencies.hooks]+ [metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]+ [metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]+ [metabase-enterprise.dependencies.task.backfill :as task.backfill]+ [metabase-enterprise.dependencies.test-util :as deps.test]+ [metabase-enterprise.serialization.test-util :as ts]+ [metabase-enterprise.serialization.v2.load :as serdes.load]+ [metabase.batch-processing.core :as grouper]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.models.serialization :as serdes]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(comment+ metabase-enterprise.dependencies.hooks/keep-me)++(defn- stale? [entity-type entity-id]+ (t2/exists? :model/DependencyStatus :entity_type entity-type :entity_id entity-id :stale true))++(defn- table-query [table]+ (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/table mp (mt/id table)))))++(defn- card-query [card-id]+ (let [mp (mt/metadata-provider)]+ (lib/query mp (lib.metadata/card mp card-id))))++(deftest card-update-test+ (testing "updating a card marks it stale only when a column its dependencies come from changes"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/DependencyStatus]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]+ (deps.dependency-status/upsert-status! :card card-id)+ (t2/update! :model/Card card-id {:name "Renamed"})+ (is (not (stale? :card card-id)))+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})+ (is (stale? :card card-id)))))))++(deftest card-update-marks-finding-test+ (testing "updating a card's query marks its own analysis finding stale in the same transaction"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/DependencyStatus :model/AnalysisFinding]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]+ (deps.analysis-finding/upsert-analysis! :card card-id true [])+ (let [finding-stale? #(t2/select-one-fn :stale :model/AnalysisFinding+ :analyzed_entity_type :card :analyzed_entity_id card-id)]+ (t2/update! :model/Card card-id {:name "Renamed"})+ (is (not (finding-stale?)))+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})+ (is (finding-stale?))))))))++(deftest card-update-without-feature-test+ (testing "without the dependencies feature the hooks do nothing"+ (mt/with-premium-features #{}+ (mt/with-model-cleanup [:model/DependencyStatus]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]+ (deps.dependency-status/upsert-status! :card card-id)+ (t2/update! :model/Card card-id {:dataset_query (table-query :orders)})+ (is (not (stale? :card card-id))))))))++(deftest dashboard-card-test+ (testing "inserting, updating and deleting a dashboard card, or adding a series, marks its dashboard stale"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/DependencyStatus]+ (mt/with-temp [:model/Dashboard {dashboard-id :id} {}+ :model/Card {card-id :id} {:dataset_query (table-query :products)}+ :model/Card {other-id :id} {:dataset_query (table-query :orders)}]+ (let [processed! #(deps.dependency-status/upsert-status! :dashboard dashboard-id)]+ (processed!)+ (let [dashcard-id (t2/insert-returning-pk! :model/DashboardCard+ {:dashboard_id dashboard-id :card_id card-id+ :row 0 :col 0 :size_x 4 :size_y 4})]+ (testing "inserting a dashboard card"+ (is (stale? :dashboard dashboard-id)))+ (processed!)+ (t2/update! :model/DashboardCard dashcard-id {:row 1})+ (testing "moving a dashboard card"+ (is (not (stale? :dashboard dashboard-id))))+ (t2/update! :model/DashboardCard dashcard-id {:card_id other-id})+ (testing "pointing a dashboard card at another card"+ (is (stale? :dashboard dashboard-id)))+ (processed!)+ (t2/insert! :model/DashboardCardSeries {:dashboardcard_id dashcard-id :card_id card-id :position 0})+ (testing "adding a series"+ (is (stale? :dashboard dashboard-id)))+ (processed!)+ (t2/delete! :model/DashboardCard :id dashcard-id)+ (testing "removing a dashboard card"+ (is (stale? :dashboard dashboard-id))))))))))++(deftest entity-delete-test+ (testing "deleting an entity marks its status stale, and the backfill drops its dependencies and status"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}]+ (t2/insert! :model/Dependency {:from_entity_type "card" :from_entity_id card-id+ :to_entity_type "table" :to_entity_id (mt/id :products)})+ (deps.dependency-status/upsert-status! :card card-id)+ (t2/delete! :model/Card :id card-id)+ (is (stale? :card card-id))+ (deps.test/synchronously-run-backfill!)+ (is (not (t2/exists? :model/Dependency :from_entity_type "card" :from_entity_id card-id)))+ (is (not (t2/exists? :model/DependencyStatus :entity_type "card" :entity_id card-id)))))))++(deftest concurrent-saves-of-dependent-cards-commit-test+ (testing "saving a card and then its dependent in one transaction, while another request saves the dependent, commits both"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/DependencyStatus]+ (let [insert-card! (fn [query]+ (t2/insert-returning-pk! :model/Card {:name (mt/random-name)+ :display :table+ :visualization_settings {}+ :creator_id (mt/user->id :crowberto)+ :database_id (mt/id)+ :dataset_query query}))+ card-a (insert-card! (table-query :products))+ card-b (insert-card! (card-query card-a))+ a-saved (promise)+ b-status-marked (promise)+ save! (fn [thunk]+ (future+ (try+ (thunk)+ :committed+ (catch Throwable e+ (ex-message (or (ex-cause e) e))))))+ mark-stale! (mt/original-fn #'deps.dependency-status/mark-stale!)]+ (t2/insert! :model/Dependency {:from_entity_type :card :from_entity_id card-b+ :to_entity_type :card :to_entity_id card-a})+ (doseq [card-id [card-a card-b]]+ (deps.dependency-status/upsert-status! :card card-id)+ (deps.analysis-finding/upsert-analysis! :card card-id true []))+ (mt/with-dynamic-fn-redefs [deps.dependency-status/mark-stale!+ (fn [type->ids]+ (mark-stale! type->ids)+ (when (= {:card #{card-b}} type->ids)+ (deliver b-status-marked true)))]+ (let [cascade (save! #(t2/with-transaction [_conn]+ (t2/update! :model/Card card-a {:dataset_query (table-query :orders)})+ (deliver a-saved true)+ (deref b-status-marked 10000 nil)+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 5)})))+ single (save! #(do (deref a-saved 10000 nil)+ (t2/update! :model/Card card-b {:dataset_query (lib/limit (card-query card-a) 11)})))]+ (is (= [:committed :committed]+ [(deref cascade 30000 :timeout) (deref single 30000 :timeout)])))))))))++(deftest serdes-load-triggers-jobs-once-test+ (testing "a serdes load marks what it writes, nested deletes included, and triggers the jobs once it finishes"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/DependencyStatus]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}+ :model/Dashboard {dashboard-id :id} {}+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id}]+ (let [exported (ts/extract-one "Dashboard" dashboard-id)+ submitted (atom [])+ triggered (atom 0)]+ (t2/insert! :model/DashboardCard {:dashboard_id dashboard-id :card_id card-id+ :row 4 :col 0 :size_x 4 :size_y 4})+ (deps.dependency-status/upsert-status! :dashboard dashboard-id)+ (mt/with-dynamic-fn-redefs [grouper/submit! (fn [_queue write] (swap! submitted conj write))+ task.backfill/trigger-backfill-job! (fn [] (swap! triggered inc))]+ (deps.test/do-with-immediate-hook-writes+ #(serdes/with-cache+ (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))+ (testing "deleting the dashboard card missing from the export marks its dashboard"+ (is (= 1 (t2/count :model/DashboardCard :dashboard_id dashboard-id)))+ (is (stale? :dashboard dashboard-id)))+ (testing "the load triggers the jobs once, after it finishes"+ (is (= [] @submitted))+ (is (= 1 @triggered)))))))))++(deftest serdes-reload-of-unchanged-dashboard-test+ (testing "reloading an unchanged dashboard export, including empty visualization settings, does not mark it"+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/DependencyStatus]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (table-query :products)}+ :model/Dashboard {dashboard-id :id} {}+ :model/DashboardCard _ {:dashboard_id dashboard-id :card_id card-id+ :visualization_settings {}}]+ (let [exported (ts/extract-one "Dashboard" dashboard-id)]+ (deps.dependency-status/upsert-status! :dashboard dashboard-id)+ (mt/with-dynamic-fn-redefs [task.backfill/trigger-backfill-job! (fn [])]+ (deps.test/do-with-immediate-hook-writes+ #(serdes/with-cache+ (serdes.load/load-metabase! (ts/ingestion-in-memory [exported]) :reindex? false))))+ (is (not (stale? :dashboard dashboard-id)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.cljdeleted file mode 100644index a6a9f1e5ce3..00000000000--- a/enterprise/backend/test/metabase_enterprise/dependencies/metadata_update_test.clj+++ /dev/null@@ -1,419 +0,0 @@-(ns metabase-enterprise.dependencies.metadata-update-test- (:require- [clojure.java.jdbc :as jdbc]- [clojure.test :refer [deftest is testing]]- [java-time.api :as t]- [metabase-enterprise.dependencies.async :as async]- [metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]- [metabase-enterprise.dependencies.events :as deps.events]- [metabase-enterprise.dependencies.findings :as deps.findings]- [metabase-enterprise.dependencies.metadata-update :as deps.metadata-update]- [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]- [metabase.api.common :as api]- [metabase.events.core :as events]- [metabase.lib-be.core :as lib-be]- [metabase.lib.core :as lib]- [metabase.lib.metadata :as lib.metadata]- [metabase.queries.card-schema :as card-schema]- [metabase.sync.core :as sync]- [metabase.test :as mt]- [metabase.test.data.one-off-dbs :as one-off-dbs]- [metabase.util :as u]- [toucan2.core :as t2]))--(defn assert-traverses-grandchild [{:keys [grandchild-dependency-type grandchild-data should-traverse? should-include?]}]- (let [mp (mt/metadata-provider)- query (->> (mt/id :products)- (lib.metadata/table mp)- (lib/query mp))- grandchild-model (deps.dependency-types/dependency-type->model grandchild-dependency-type)]- (mt/with-temp [:model/Card {parent-id :id} {:dataset_query query}- :model/Card {child-id :id} {:dataset_query query}- grandchild-model {grandchild-id :id} grandchild-data- :model/Card {great-grandchild-id :id} {:dataset_query query}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}- :model/Dependency _ {:from_entity_type grandchild-dependency-type- :from_entity_id grandchild-id- :to_entity_type :card- :to_entity_id child-id}- :model/Dependency _ {:from_entity_type :card- :from_entity_id great-grandchild-id- :to_entity_type grandchild-dependency-type- :to_entity_id grandchild-id}]- (is (= (cond-> #{child-id}- should-include? (conj grandchild-id)- should-traverse? (conj great-grandchild-id))- (set (#'deps.metadata-update/dependent-mbql-cards- (#'deps.metadata-update/mbql-graph mp)- :card parent-id)))))))--(deftest ^:parallel dependent-mbql-cards-traverses-cards-test- (testing "dependent-mbql-cards looks through card dependencies"- (let [mp (mt/metadata-provider)- query (->> (mt/id :products)- (lib.metadata/table mp)- (lib/query mp))]- (assert-traverses-grandchild- {:grandchild-dependency-type :card- :grandchild-data {:dataset_query query}- :should-traverse? true- :should-include? true}))))--(deftest ^:parallel dependent-mbql-cards-stops-on-native-cards-test- (testing "dependent-mbql-cards stops when it reaches a native card"- (let [mp (mt/metadata-provider)- native-query (lib/native-query mp "select * from orders")]- (assert-traverses-grandchild- {:grandchild-dependency-type :card- :grandchild-data {:dataset_query native-query}}))))--(deftest ^:parallel dependent-mbql-cards-stops-on-snippets-test- (testing "dependent-mbql-cards stops when it reaches a snippet"- (assert-traverses-grandchild- {:grandchild-dependency-type :snippet- :grandchild-data {:name "test snippet"- :content "SELECT 1"}})))--(deftest ^:parallel dependent-mbql-cards-stops-on-transforms-test- (testing "dependent-mbql-cards stops when it reaches a transform"- (assert-traverses-grandchild- {:grandchild-dependency-type :transform- :grandchild-data {}})))--(deftest ^:parallel dependent-mbql-cards-stops-on-dashboards-test- (testing "dependent-mbql-cards stops when it reaches a dashboard"- (assert-traverses-grandchild- {:grandchild-dependency-type :dashboard- :grandchild-data {}})))--(deftest ^:parallel dependent-mbql-cards-stops-on-documents-test- (testing "dependent-mbql-cards stops when it reaches a document"- (assert-traverses-grandchild- {:grandchild-dependency-type :document- :grandchild-data {}})))--(deftest ^:parallel dependent-mbql-cards-traverses-through-segments-test- (testing "dependent-mbql-cards traverses through segments"- (assert-traverses-grandchild- {:grandchild-dependency-type :segment- :grandchild-data {}- :should-traverse? true})))--(deftest ^:parallel dependent-mbql-cards-traverses-through-measures-test- (testing "dependent-mbql-cards traverses through measures"- (assert-traverses-grandchild- {:grandchild-dependency-type :measure- :grandchild-data {}- :should-traverse? true})))--(deftest ^:synchronized card-update-updates-child-metadata-test- (testing "card updates update child card metadata"- (mt/with-test-user :rasta- (mt/with-premium-features #{:dependencies}- (let [mp (mt/metadata-provider)- products-id (mt/id :products)- orders-id (mt/id :orders)- products (lib.metadata/table mp products-id)- orders (lib.metadata/table mp orders-id)]- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)}- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}- :model/Dependency _ {:from_entity_type :card- :from_entity_id grandchild-id- :to_entity_type :card- :to_entity_id child-id}]- (is (= #{8}- (t2/select-fn-set (comp count :result_metadata)- (card-schema/selection [:database_id])- :id [:in [parent-id child-id grandchild-id]])))- (t2/update! :model/Card parent-id {:dataset_query (lib/query mp orders)})- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update- {:object (assoc parent-card :dataset_query (lib/query mp orders))- :previous-object parent-card- :user-id api/*current-user-id*}))- (is (= #{9}- (t2/select-fn-set (comp count :result_metadata)- (card-schema/selection [:database_id])- :id [:in [parent-id child-id grandchild-id]])))))))))--(deftest ^:synchronized native-card-update-does-not-update-children-test- (testing "native card updates do not update children"- (mt/with-test-user :rasta- (mt/with-premium-features #{:dependencies}- (let [mp (mt/metadata-provider)- products-id (mt/id :products)- products (lib.metadata/table mp products-id)- native-query (lib/native-query mp "select * from orders")]- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)}- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}]- (t2/update! :model/Card child-id {:result_metadata nil})- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update- {:object (assoc parent-card :dataset_query native-query)- :previous-object parent-card- :user-id api/*current-user-id*}))- (is (= nil- (t2/select-one-fn :result_metadata- (card-schema/selection [:database_id])- :id child-id)))))))))--(deftest ^:synchronized model-update-passes-down-new-values-test- (testing "model updates pass down new result metadata"- (mt/with-test-user :rasta- (mt/with-premium-features #{:dependencies}- (let [mp (mt/metadata-provider)- products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)- :type :model}- :model/Card {child-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))-- :type :model}- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))- :type :model}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}- :model/Dependency _ {:from_entity_type :card- :from_entity_id grandchild-id- :to_entity_type :card- :to_entity_id child-id}]- (let [new-result-metadata (assoc-in (:result_metadata parent-card)- [0 :display_name]- "new-name")]- (t2/update! :model/Card parent-id {:result_metadata new-result-metadata})- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update- {:object (assoc parent-card :result_metadata new-result-metadata)- :previous-object parent-card- :user-id api/*current-user-id*}))- (is (= #{[child-id "new-name"] [grandchild-id "new-name"]}- (t2/select-fn-set (juxt :id #(get-in % [:result_metadata 0 :display_name]))- (card-schema/selection [:database_id])- :id [:in [child-id grandchild-id]]))))))))))--(deftest ^:synchronized model-update-respects-child-overrides-test- (testing "model updates respect child metadata edits"- (mt/with-test-user :rasta- (mt/with-premium-features #{:dependencies}- (let [mp (mt/metadata-provider)- products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)- :type :model}- :model/Card {child-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))- :type :model}- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))- :type :model}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}- :model/Dependency _ {:from_entity_type :card- :from_entity_id grandchild-id- :to_entity_type :card- :to_entity_id child-id}]- (let [new-parent-metadata (assoc-in (:result_metadata parent-card)- [0 :display_name]- "new-name")- new-child-metadata (assoc-in (:result_metadata child-card)- [0 :display_name]- "child-name")- new-grandchild-metadata (assoc-in (:result_metadata child-card)- [0 :display_name]- "grandchild-name")]- (t2/update! :model/Card parent-id {:result_metadata new-parent-metadata})- (t2/update! :model/Card child-id {:result_metadata new-child-metadata})- (t2/update! :model/Card grandchild-id {:result_metadata new-grandchild-metadata})- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update- {:object (assoc parent-card :result_metadata new-parent-metadata)- :previous-object parent-card- :user-id api/*current-user-id*}))- (is (= #{[child-id "child-name"] [grandchild-id "grandchild-name"]}- (t2/select-fn-set (juxt :id #(get-in % [:result_metadata 0 :display_name]))- (card-schema/selection [:database_id])- :id [:in [child-id grandchild-id]]))))))))))--(deftest ^:synchronized model-update-stops-recursing-when-child-metadata-is-unchanged-test- (testing "model updates stop recursing when they hit a child whose metadata didn't change"- (mt/with-test-user :rasta- (mt/with-premium-features #{:dependencies}- (let [mp (mt/metadata-provider)- products-id (mt/id :products)- products (lib.metadata/table mp products-id)]- (mt/with-temp [:model/Card {parent-id :id :as parent-card} {:dataset_query (lib/query mp products)- :type :model}- :model/Card {child-id :id :as child-card} {:dataset_query (lib/query mp (lib.metadata/card mp parent-id))- :type :model}- :model/Card {grandchild-id :id} {:dataset_query (lib/query mp (lib.metadata/card mp child-id))- :type :model}- :model/Dependency _ {:from_entity_type :card- :from_entity_id child-id- :to_entity_type :card- :to_entity_id parent-id}- :model/Dependency _ {:from_entity_type :card- :from_entity_id grandchild-id- :to_entity_type :card- :to_entity_id child-id}]- (let [new-parent-metadata (assoc-in (:result_metadata parent-card)- [0 :display_name]- "new-name")- new-child-metadata (-> (:result_metadata child-card)- (assoc-in [0 :display_name]- "child-name")- (assoc-in [0 :lib/original-display-name]- "new-name"))]- (t2/update! :model/Card parent-id {:result_metadata new-parent-metadata})- (t2/update! :model/Card child-id {:result_metadata new-child-metadata})- (t2/update! :model/Card grandchild-id {:result_metadata nil})- (mt/with-dynamic-fn-redefs [async/submit! (fn [f] (f))]- (events/publish-event! :event/card-update- {:object (assoc parent-card :result_metadata new-parent-metadata)- :previous-object parent-card- :user-id api/*current-user-id*}))- (is (= nil- (t2/select-one-fn #(get-in % [:result_metadata 0 :display_name])- (card-schema/selection [:database_id])- :id grandchild-id))))))))))--(defn- with-syncable-db! [thunk]- (mt/with-premium-features #{:dependencies}- (one-off-dbs/with-blank-db- ;; Step 1: Create a table with two columns- (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"- "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]- (jdbc/execute! one-off-dbs/*conn* [statement]))- ;; Sync the database to pick up the new table- (sync/sync-database! (mt/db))- (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")- filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")- ;; Step 2: Create a card with a filter on filter_col- mp (lib-be/application-database-metadata-provider (mt/id))- table (lib.metadata/table mp table-id)- filter-field (lib.metadata/field mp filter-field-id)- query (-> (lib/query mp table)- (lib/filter (lib/= filter-field "value1")))]- (mt/with-temp [:model/Card {card-id :id} {:dataset_query query- :database_id (mt/id)}]- ;; Create the dependency manually- (t2/insert! :model/Dependency {:from_entity_type :card- :from_entity_id card-id- :to_entity_type :table- :to_entity_id table-id})- ;; Step 3: Run analysis - should succeed with no errors- (let [card (t2/select-one :model/Card card-id)]- ;; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see- ;; old values after the sync!- (lib-be/with-metadata-provider-cache- (deps.findings/upsert-analysis! card)))- (testing "Initial analysis succeeds"- (is (true?- (t2/select-one-fn :result :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)))- (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))- ;; Backdate the analysis timestamp and table/field updated_at so- ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)- ;; after a new sync makes an edit.- ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and- ;; the field update from sync would have identical timestamps.- (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]- (t2/update! :model/AnalysisFinding- {:analyzed_entity_type :card :analyzed_entity_id card-id}- {:analyzed_at old-time})- (t2/update! :model/Field :table_id table-id- {:updated_at old-time})- (t2/update! :model/Table :id table-id- {:updated_at old-time}))- ;; Setup complete: Now call the thunk for the actual test run.- (thunk {:db-id (:id (mt/db))- :card-id card-id- :table-id table-id- :filter-field-id filter-field-id}))))))--;; Integration test that a real DB sync triggers re-analysis of an updated table.-(deftest ^:synchronized sync-removed-column-triggers-reanalysis-test- (testing "When sync detects a removed column, re-analyzing the card shows errors"- (with-syncable-db!- (fn [{:keys [card-id filter-field-id]}]- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- ;; Step 4: Remove the column used in the filter- (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])- ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis- ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.- (sync/sync-database! (mt/db))- ;; Verify the field is now inactive- (testing "Field is marked inactive after sync"- (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))- ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.- ;; Due to race conditions, the entity may be either stale (job hasn't run yet)- ;; or already reanalyzed (job ran). Check for either valid outcome.- (let [finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- (is (some? finding) "Analysis finding should exist")- (if (:stale finding)- ;; Job hasn't run yet - sync marked it stale, that's the expected behavior- (testing "Entity is marked stale for re-analysis"- (is (true? (:stale finding))))- ;; Job already ran - verify the reanalysis detected the missing column- (testing "Re-analysis detects the missing column"- (is (false? (:result finding)) "Analysis should fail")- (is (not= initial-analysis-time (:analyzed_at finding))- "Analysis should have been re-run")- (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]- (is (seq errors) "Should have analysis errors")- (is (some #(= :missing-column (:error_type %)) errors)- "Should have a missing-column error"))))))))))--;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all-;; cards which depend on that table.-(deftest ^:synchronized sync-without-changes-does-not-trigger-reanalysis-test- (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"- (with-syncable-db!- (fn [{:keys [card-id db-id filter-field-id table-id]}]- (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)- db-deps-checked (atom [])- original-deps-check (mt/original-fn #'deps.events/synced-db->direct-dependents-of-changed-tables)- table-before (into {} (t2/select-one :model/Table :id table-id))- filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]- ;; Re-sync the database, having made no changes to it.- (mt/with-dynamic-fn-redefs [deps.events/synced-db->direct-dependents-of-changed-tables- (fn [db-id]- (u/prog1 (original-deps-check db-id)- (swap! db-deps-checked conj [db-id <>])))]- (sync/sync-database! (mt/db)))- (testing "sync doesn't update tables or fields that haven't changed"- (let [table-after (into {} (t2/select-one :model/Table :id table-id))- filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]- (is (= table-before table-after))- (is (= filter-field-before filter-field-after))))- (testing "the DB is were checked for tables that need deps updates, but none were found"- (is (=? [[db-id empty?]]- @db-deps-checked)))- (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"- (let [finding (t2/select-one :model/AnalysisFinding- :analyzed_entity_type :card- :analyzed_entity_id card-id)]- (is (some? finding) "Analysis finding should exist")- (is (true? (:result finding)) "Analysis should be a success")- (is (= initial-analysis-time (:analyzed_at finding))- "Analysis should not have been re-run"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.cljindex dcefe1b4346..6c2e4ab094f 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/dependency_test.clj@@ -185,7 +185,7 @@(upstream-of :card (:id question-card))))))))))))(deftest dependency-status-marked-stale-test- (testing "dependency_status is marked stale when entities are created or updated"+ (testing "dependency_status is marked stale when entities are updated, and not touched when they are created"(mt/with-model-cleanup [:model/DependencyStatus](testing "cards"(mt/with-model-cleanup [:model/Dependency]@@ -197,7 +197,7 @@(testing "transforms create"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/Transform transform]- (is (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform) :stale true)))))+ (is (not (t2/exists? :model/DependencyStatus :entity_type :transform :entity_id (:id transform)))))))(testing "transforms update"(mt/with-premium-features #{}(mt/with-temp [:model/Transform transform]@@ -207,7 +207,7 @@(testing "snippets create"(mt/with-premium-features #{:dependencies}(mt/with-temp [:model/NativeQuerySnippet snippet]- (is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true)))))+ (is (not (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet)))))))(testing "snippets update"(mt/with-premium-features #{}(mt/with-temp [:model/NativeQuerySnippet snippet]@@ -215,13 +215,15 @@(t2/update! :model/NativeQuerySnippet (:id snippet) {:content "new content"}))(is (t2/exists? :model/DependencyStatus :entity_type :snippet :entity_id (:id snippet) :stale true))))))))-(deftest dependency-status-marked-stale-on-card-create-test- (testing "dependency_status is marked stale when a card is created"+(deftest dependency-status-absent-until-backfill-on-card-create-test+ (testing "a created card has no dependency_status row until the backfill job processes it"(mt/with-premium-features #{:dependencies}(mt/with-model-cleanup [:model/Card :model/Dependency :model/DependencyStatus](mt/with-temp [:model/User user](let [card (card/create-card! (basic-orders) user)]- (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale true))))))))+ (is (not (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card))))+ (deps.test/synchronously-run-backfill!)+ (is (t2/exists? :model/DependencyStatus :entity_type :card :entity_id (:id card) :stale false))))))))(deftest filtered-graph-dependencies-test(testing "filtered-graph-dependencies respects filter clause"diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.cljnew file mode 100644index 00000000000..956d3f0e5bc--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/dependencies/sync_test.clj@@ -0,0 +1,158 @@+(ns metabase-enterprise.dependencies.sync-test+ (:require+ [clojure.java.jdbc :as jdbc]+ [clojure.test :refer [deftest is testing]]+ [java-time.api :as t]+ [metabase-enterprise.dependencies.db :as deps.db]+ [metabase-enterprise.dependencies.findings :as deps.findings]+ [metabase-enterprise.dependencies.hooks]+ [metabase-enterprise.dependencies.models.analysis-finding-error :as deps.analysis-finding-error]+ [metabase.lib-be.core :as lib-be]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.sync.core :as sync]+ [metabase.test :as mt]+ [metabase.test.data.one-off-dbs :as one-off-dbs]+ [metabase.util :as u]+ [toucan2.core :as t2]))++(defn- with-syncable-db! [thunk]+ (mt/with-premium-features #{:dependencies}+ (one-off-dbs/with-blank-db+ ;; Step 1: Create a table with two columns+ (doseq [statement ["CREATE TABLE \"test_table\" (\"id\" INTEGER PRIMARY KEY, \"filter_col\" VARCHAR);"+ "INSERT INTO \"test_table\" (\"id\", \"filter_col\") VALUES (1, 'value1'), (2, 'value2');"]]+ (jdbc/execute! one-off-dbs/*conn* [statement]))+ ;; Sync the database to pick up the new table+ (sync/sync-database! (mt/db))+ (let [table-id (t2/select-one-fn :id :model/Table :db_id (mt/id) :name "test_table")+ filter-field-id (t2/select-one-fn :id :model/Field :table_id table-id :name "filter_col")+ ;; Step 2: Create a card with a filter on filter_col+ mp (lib-be/application-database-metadata-provider (mt/id))+ table (lib.metadata/table mp table-id)+ filter-field (lib.metadata/field mp filter-field-id)+ query (-> (lib/query mp table)+ (lib/filter (lib/= filter-field "value1")))]+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query query+ :database_id (mt/id)}]+ ;; Create the dependency manually+ (t2/insert! :model/Dependency {:from_entity_type :card+ :from_entity_id card-id+ :to_entity_type :table+ :to_entity_id table-id})+ ;; Step 3: Run analysis - should succeed with no errors+ (let [card (t2/select-one :model/Card card-id)]+ ;; NOTE: The metadata provider caches must be small here - if the cache spans a sync it will see+ ;; old values after the sync!+ (lib-be/with-metadata-provider-cache+ (deps.findings/upsert-analysis! card)))+ (testing "Initial analysis succeeds"+ (is (true?+ (t2/select-one-fn :result :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)))+ (is (empty? (deps.analysis-finding-error/errors-for-entity :card card-id))))+ ;; Backdate the analysis timestamp and table/field updated_at so+ ;; synced-db->direct-dependents-of-changed-tables will detect it as stale (analyzed_at < field.updated_at)+ ;; after a new sync makes an edit.+ ;; This is necessary because within a transaction, now() returns the same value, so the initial analysis and+ ;; the field update from sync would have identical timestamps.+ (let [old-time (t/minus (t/offset-date-time) (t/hours 1))]+ (t2/update! :model/AnalysisFinding+ {:analyzed_entity_type :card :analyzed_entity_id card-id}+ {:analyzed_at old-time})+ (t2/update! :model/Field :table_id table-id+ {:updated_at old-time})+ (t2/update! :model/Table :id table-id+ {:updated_at old-time}))+ ;; Setup complete: Now call the thunk for the actual test run.+ (thunk {:db-id (:id (mt/db))+ :card-id card-id+ :table-id table-id+ :filter-field-id filter-field-id}))))))++;; Integration test that a real DB sync triggers re-analysis of an updated table.+(deftest ^:synchronized sync-removed-column-triggers-reanalysis-test+ (testing "When sync detects a removed column, re-analyzing the card shows errors"+ (with-syncable-db!+ (fn [{:keys [card-id filter-field-id]}]+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ ;; Step 4: Remove the column used in the filter+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])+ ;; Step 5: Re-sync the database. The sync-end event will detect the stale analysis+ ;; (because analyzed_at < field.updated_at) and mark it for re-analysis.+ (sync/sync-database! (mt/db))+ ;; Verify the field is now inactive+ (testing "Field is marked inactive after sync"+ (is (false? (t2/select-one-fn :active :model/Field :id filter-field-id))))+ ;; Step 6 & 7: The sync-end event marks dependents as stale and triggers the job.+ ;; Due to race conditions, the entity may be either stale (job hasn't run yet)+ ;; or already reanalyzed (job ran). Check for either valid outcome.+ (let [finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ (is (some? finding) "Analysis finding should exist")+ (if (:stale finding)+ ;; Job hasn't run yet - sync marked it stale, that's the expected behavior+ (testing "Entity is marked stale for re-analysis"+ (is (true? (:stale finding))))+ ;; Job already ran - verify the reanalysis detected the missing column+ (testing "Re-analysis detects the missing column"+ (is (false? (:result finding)) "Analysis should fail")+ (is (not= initial-analysis-time (:analyzed_at finding))+ "Analysis should have been re-run")+ (let [errors (deps.analysis-finding-error/errors-for-entity :card card-id)]+ (is (seq errors) "Should have analysis errors")+ (is (some #(= :missing-column (:error_type %)) errors)+ "Should have a missing-column error"))))))))))++(deftest ^:synchronized scheduled-metadata-sync-triggers-reanalysis-test+ (testing "The metadata sync the scheduled sync task runs marks the dependents of a changed table for re-analysis"+ (with-syncable-db!+ (fn [{:keys [card-id]}]+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ (jdbc/execute! one-off-dbs/*conn* ["ALTER TABLE \"test_table\" DROP COLUMN \"filter_col\";"])+ (sync/sync-db-metadata! (mt/db))+ (let [finding (t2/select-one :model/AnalysisFinding :analyzed_entity_type :card :analyzed_entity_id card-id)]+ (is (or (:stale finding)+ (not= initial-analysis-time (:analyzed_at finding))))))))))++;; Integration test that a DB sync which doesn't change anything about a table does not trigger re-analysis of all+;; cards which depend on that table.+(deftest ^:synchronized sync-without-changes-does-not-trigger-reanalysis-test+ (testing "When sync makes no changes to a table or its fields, the card is not re-analyzed"+ (with-syncable-db!+ (fn [{:keys [card-id db-id filter-field-id table-id]}]+ (let [initial-analysis-time (t2/select-one-fn :analyzed_at :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)+ db-deps-checked (atom [])+ original-deps-check (mt/original-fn #'deps.db/table-ids-with-outdated-findings)+ table-before (into {} (t2/select-one :model/Table :id table-id))+ filter-field-before (into {} (t2/select-one :model/Field :id filter-field-id))]+ ;; Re-sync the database, having made no changes to it.+ (mt/with-dynamic-fn-redefs [deps.db/table-ids-with-outdated-findings+ (fn [db-id]+ (u/prog1 (original-deps-check db-id)+ (swap! db-deps-checked conj [db-id <>])))]+ (sync/sync-database! (mt/db)))+ (testing "sync doesn't update tables or fields that haven't changed"+ (let [table-after (into {} (t2/select-one :model/Table :id table-id))+ filter-field-after (into {} (t2/select-one :model/Field :id filter-field-id))]+ (is (= table-before table-after))+ (is (= filter-field-before filter-field-after))))+ (testing "the DB is were checked for tables that need deps updates, but none were found"+ (is (=? [[db-id empty?]]+ @db-deps-checked)))+ (testing "No re-analysis was done, the analysis for the card is ~1 hour ago"+ (let [finding (t2/select-one :model/AnalysisFinding+ :analyzed_entity_type :card+ :analyzed_entity_id card-id)]+ (is (some? finding) "Analysis finding should exist")+ (is (true? (:result finding)) "Analysis should be a success")+ (is (= initial-analysis-time (:analyzed_at finding))+ "Analysis should not have been re-run"))))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljindex 58deb44af0f..e84d4e5159f 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj@@ -1,6 +1,7 @@(ns metabase-enterprise.dependencies.task.backfill-test{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.dependencies.task.backfill-test]}}}}}}(:require+ [clojure.string :as str][clojure.test :refer :all][environ.core :as env][java-time.api :as t]@@ -9,7 +10,7 @@[metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status][metabase-enterprise.dependencies.task.backfill :as dependencies.backfill][metabase-enterprise.dependencies.test-util :as deps.test]- [metabase.events.core :as events]+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.premium-features.core :as premium-features][metabase.task.core :as task][metabase.test :as mt]@@ -34,7 +35,7 @@(defn- mark-stale!"Mark an entity as stale in dependency_status for testing."[entity-type entity-id]- (deps.dependency-status/mark-stale! entity-type [entity-id]))+ (deps.dependency-status/mark-stale! {entity-type #{entity-id}}))(defn- assert-processed"Assert that an entity has been processed (not stale, current version)."@@ -374,33 +375,6 @@(assert-processed :card card-id)(is (= 1 deps-after))))))))-(deftest ^:synchronized backfill-dependencies-on-serdes-load-test- (testing "Serdes load triggers the backfill job (entities remain stale until job runs)"- (backfill-all-existing-entities!)- (let [query (mt/mbql-query orders)]- (mt/with-premium-features #{:dependencies}- (mt/with-temp [:model/Card {card1-id :id} {:dataset_query query}- :model/Card {card2-id :id} {:dataset_query query}- :model/Card {card3-id :id} {:dataset_query query}]- ;; Mark all as stale- (mark-stale! :card card1-id)- (mark-stale! :card card2-id)- (mark-stale! :card card3-id)- ;; serdes-load triggers the async job, not synchronous processing- (events/publish-event! :event/serdes-load {})- ;; entities are still stale (job hasn't run yet in this test)- (is (= 3 (t2/count :model/DependencyStatus- :entity_type :card- :entity_id [:in [card1-id card2-id card3-id]]- :stale true)))- ;; manually run the backfill to verify it processes them- (backfill-all-existing-entities!)- (is (= 3 (t2/count :model/DependencyStatus- :entity_type :card- :entity_id [:in [card1-id card2-id card3-id]]- :stale false- :dependency_analysis_version dependencies.model/current-dependency-analysis-version))))))))-(deftest ^:synchronized backfill-dependencies-on-token-update-test(testing "Token update triggers the backfill job (entities remain stale until job runs)"(backfill-all-existing-entities!)@@ -414,7 +388,7 @@(mark-stale! :card card2-id)(mark-stale! :card card3-id);; token update triggers the async job, not synchronous processing- (events/publish-event! :event/set-premium-embedding-token {})+ (dependencies.hooks/after-premium-token-update);; entities are still stale (job hasn't run yet in this test)(is (= 3 (t2/count :model/DependencyStatus:entity_type :card@@ -485,7 +459,10 @@(reset! scheduled [])(thunk)(count @scheduled))]- (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)]+ (mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args]+ (when (some #(str/includes? (str %) "dependency-backfill") args)+ (swap! scheduled conj args))+ nil)](with-redefs [env/env (assoc env/env :mb-dependency-backfill-batch-size "0")](testing "the 1-second event-driven trigger is suppressed"(is (zero? (scheduled-by dependencies.backfill/trigger-backfill-job!))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.cljindex a7c89c87e99..aa770887e24 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/entity_check_test.clj@@ -3,7 +3,7 @@[clojure.test :refer [deftest is testing]][environ.core :as env][metabase-enterprise.dependencies.task.entity-check :as dependencies.entity-check]- [metabase.events.core :as events]+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.premium-features.core :as premium-features][metabase.task.core :as task][metabase.test :as mt])@@ -61,18 +61,15 @@(def ^:private entity-check-job-name "metabase.dependencies.task.entity-check.job")(defn- scheduled-job-names- "Job names from captured [[task/schedule-task!]] calls. Needed because several jobs listen for the same events, so a- bare call count cannot tell us which job got scheduled."+ "Job names from captured [[task/schedule-task!]] calls, since a token update schedules both dependency jobs."[calls](into #{} (map (fn [args] (.getName (.getKey ^JobDetail (nth args 1))))) calls))-(deftest ^:synchronized token-event-restarts-the-job-test- (testing "Setting a premium token must restart the entity check job. The backfill job derives- :event/set-premium-embedding-token for this; entity check had no event wiring at all, so its only restarts- came from content-change handlers. Enabling the feature on an idle instance left the periodic checker dead- until the next process restart."+(deftest ^:synchronized token-update-restarts-the-job-test+ (testing "Setting a premium token must restart the entity check job, or enabling the feature on an idle instance+ leaves the periodic checker dead until the next process restart."(let [scheduled (atom [])](mt/with-dynamic-fn-redefs [task/schedule-task! (fn [& args] (swap! scheduled conj args) nil)](mt/with-premium-features #{:dependencies}- (events/publish-event! :event/set-premium-embedding-token {})+ (dependencies.hooks/after-premium-token-update)(is (contains? (scheduled-job-names @scheduled) entity-check-job-name)))))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.cljindex b122e09b05c..a61b623fa59 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/test_util.clj@@ -2,6 +2,7 @@(:require[medley.core :as m][metabase-enterprise.dependencies.task.backfill :as dependencies.backfill]+ [metabase.app-db.core :as mdb][metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]@@ -16,6 +17,13 @@(mt/with-premium-features #{:dependencies}(while (#'dependencies.backfill/backfill-dependencies!))))+(defn do-with-immediate-hook-writes+ "Run `thunk` with the work the dependency hooks defer until commit done at once, since a test's transaction never+ commits."+ [thunk]+ (mt/with-dynamic-fn-redefs [mdb/do-after-commit (fn [f] (f))]+ (thunk)))+(defn mock-card [metadata-provider {:keys [id query details]}](merge {:lib/type :metadata/card:id iddiff --git a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/api_test.cljindex e13d5e4f5a4..f9fe5207c5a 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/api_test.clj@@ -1,7 +1,7 @@(ns ^:mb/driver-tests metabase-enterprise.replacement.api-test(:require[clojure.test :refer :all]- [metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.hooks][metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.execute :as replacement.execute][metabase-enterprise.replacement.models.replacement-run :as replacement-run]diff --git a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.cljindex 75a891c0182..61704ca4d83 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/runner_test.clj@@ -3,7 +3,7 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase-enterprise.replacement.runner-test]}}}}}}(:require[clojure.test :refer [deftest is testing]]- [metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.hooks][metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.protocols :as replacement.protocols][metabase-enterprise.replacement.runner :as replacement.runner]@@ -20,9 +20,6 @@(set! *warn-on-reflection* true)-(comment- metabase-enterprise.dependencies.events/keep-me)-(deftest bulk-load-metadata-for-entities-test(testing "bulk-load-metadata-for-entities! fetches all entity types in bulk"(mt/with-temp [:model/Card {card-id-1 :id} {:name "Card 1"diff --git a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.cljindex 8a4c0ea27ba..bdb98152c40 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/source_swap_test.clj@@ -2,7 +2,7 @@(:require[clojure.test :refer [deftest is testing]][medley.core :as m]- [metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.hooks][metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.field-refs :as replacement.field-refs][metabase-enterprise.replacement.source-swap :as replacement.source-swap]@@ -15,9 +15,6 @@(set! *warn-on-reflection* true)-(comment- metabase-enterprise.dependencies.events/keep-me)-;;; ------------------------------------------------ Card --------------------------------------------------------(deftest card-swap-source!-source-table-testdiff --git a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.cljindex 59f9ad7d421..487366aa058 100644--- a/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj+++ b/enterprise/backend/test/metabase_enterprise/replacement/usages_test.clj@@ -1,7 +1,7 @@(ns metabase-enterprise.replacement.usages-test(:require[clojure.test :refer [deftest is testing]]- [metabase-enterprise.dependencies.events]+ [metabase-enterprise.dependencies.hooks][metabase-enterprise.dependencies.test-util :as deps.test][metabase-enterprise.replacement.usages :as replacement.usages][metabase.graph.core :as graph]@@ -12,9 +12,6 @@(set! *warn-on-reflection* true)-(comment- metabase-enterprise.dependencies.events/keep-me)-;;; ------------------------------------------------ In-memory graph helpers ------------------------------------------------(defn- deps->adjacency-mapdiff --git a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj b/enterprise/backend/test/metabase_enterprise/serialization/test_util.cljindex 5d8f905fb15..16ca6181f34 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/test_util.clj@@ -2,6 +2,7 @@(:require[clojure.java.io :as io][clojure.test :refer :all]+ [metabase-enterprise.serialization.v2.ingest :as serdes.ingest][metabase.app-db.connection :as mdb.connection][metabase.app-db.core :as mdb][metabase.app-db.schema-migrations-test.impl :as schema-migrations-test.impl]@@ -545,3 +546,12 @@:else (t2/select-fn-vec pk [model pk] {:where where}))](u/rfirst (serdes/extract-all model-name (cond-> {}ids (assoc :filter-column pk :filter-ids ids))))))++(defn ingestion-in-memory+ "An Ingestable serving the extracted `entities`, in order, under their label-free paths as the YAML ingestion does."+ [entities]+ (let [by-path (u/index-by (comp serdes.ingest/strip-labels serdes/path) entities)]+ (reify serdes.ingest/Ingestable+ (ingest-list [_] (map (comp serdes.ingest/strip-labels serdes/path) entities))+ (ingest-one [_ path] (by-path (serdes.ingest/strip-labels path)))+ (ingest-errors [_] []))))diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.cljindex fbef6cfcfdd..aebc7ad35f3 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/extract_test.clj@@ -1694,7 +1694,7 @@(map serdes/path)(into #{})))))(testing "the click behavior looks sane"- (is (= #{{:column_settings nil}+ (is (= #{{}{:column_settings {"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Other Field\"],null]]" {}"[\"ref\",[\"field\",[\"My Database\",\"PUBLIC\",\"Schema'd Table\",\"Field To Click 2\"],null]]" {}}}}(->> {:targets [["Collection" coll-id]]diff --git a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.cljindex 20171203b05..bcc717d7a4d 100644--- a/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj+++ b/enterprise/backend/test/metabase_enterprise/serialization/v2/load_test.clj@@ -34,9 +34,6 @@(binding [models.database/*include-h2-in-extract?* true](thunk)))))-(defn- no-labels [path]- (mapv #(dissoc % :label) path))-(defn- by-model [entities model-name](filter #(-> % :serdes/meta last :model (= model-name))entities))@@ -46,19 +43,6 @@(map (comp :id last :serdes/meta))set))-(defn- ingestion-in-memory [extractions]- (let [mapped (into {} (for [entity (vec extractions)]- [(no-labels (serdes/path entity))- entity]))]- (reify- serdes.ingest/Ingestable- (ingest-list [_]- (map (comp no-labels serdes/path) extractions))- (ingest-one [_ path]- (get mapped (no-labels path)))- (ingest-errors [_]- []))))-;;; WARNING for test authors: [[extract/extract]] returns a lazy reducible value. To make sure you don't;;; confound your tests with data from your dev appdb, remember to eagerly;;; `(into [] (extract/extract ...))` in these tests.@@ -104,7 +88,7 @@(let [{:keys [db coll card]} @extractedbad-card (assoc-in card [:dataset_query :stages 0 :template-tags]{"id" {:type :tag-type-from-the-future :name "id" :display-name "ID" :id "abc-123"}})- ingestion #(ingestion-in-memory [db coll bad-card])+ ingestion #(ts/ingestion-in-memory [db coll bad-card])ours? #(some (partial re-find #"does not match this Metabase's query schema") %)](testing "by default the schema check is what refuses the import"(mt/with-empty-h2-app-db!@@ -140,7 +124,7 @@(ts/create! :model/DashboardCard :dashboard_id (:id dashboard) :card_id (:id model))(reset! serialized (into [] (serdes.extract/extract {})))))(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory+ (serdes.load/load-metabase! (ts/ingestion-in-memory(for [entity @serialized](cond-> entity(= "Action" (-> entity :serdes/meta last :model))@@ -165,7 +149,7 @@@serialized))))(testing "loading into an empty database succeeds"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; the trash is serialized and loaded, so restrict to nil type(let [colls (t2/select :model/Collection :type nil)](is (= 1 (count colls)))@@ -173,7 +157,7 @@(is (= eid1 (:entity_id (first colls)))))))(testing "loading again into the same database does not duplicate"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [colls (t2/select :model/Collection :type nil)](is (= 1 (count colls)))(is (= "Basic Collection" (:name (first colls))))@@ -213,7 +197,7 @@age (ts/create! :model/Field :name "age" :table_id (:id table))email (ts/create! :model/Field :name "email" :table_id (:id table))](ts/create! :model/Dimension :field_id (:id email) :name "Local" :type :internal)- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (= "Renamed" (t2/select-one-fn :display_name :model/TableUserSettings :table_id (:id table))))(is (= "edited" (t2/select-one-fn :description :model/FieldUserSettings :field_id (:id age))))(testing "the imported Dimension replaces the Field's local one"@@ -239,7 +223,7 @@(into (extract "FieldUserSettings" :field_id [(:id age)]))(into (extract "Dimension" :field_id [(:id email)]))))))(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [stub (t2/select-one :model/Database :name "source-only-db")table (t2/select-one :model/Table :db_id (:id stub) :name "customers")](is (true? (:is_stub stub)))@@ -264,7 +248,7 @@(t2/insert! :model/FieldUserSettings {:field_id name-id :semantic_type :type/City})(extract [price-id name-id]))](try- (serdes.load/load-metabase! (ingestion-in-memory serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory serialized))(let [stub (t2/select-one :model/Database :name db-name)table-id (t2/select-one-pk :model/Table :db_id (:id stub) :schema "PUBLIC" :name "VENUES")field-ids (t2/select-pks-vec :model/Field :table_id table-id {:order-by [[:name :asc]]})@@ -308,7 +292,7 @@(serdes/extract-one "Card" {} good)])))(ts/with-db dest-db(serdes/with-cache- (serdes.load/load-metabase! (ingestion-in-memory (update @serialized 1 assoc :name nil))+ (serdes.load/load-metabase! (ts/ingestion-in-memory (update @serialized 1 assoc :name nil)):continue-on-error true))(let [stub (t2/select-one :model/Database :name "source-only-db")](is (true? (:is_stub stub)))@@ -326,7 +310,7 @@:name "Email" :type :internal))):serdes/meta)field (ts/extract-one "Field" (:id email))- load! (fn [entity] (serdes.load/load-metabase! (ingestion-in-memory [entity])))]+ load! (fn [entity] (serdes.load/load-metabase! (ts/ingestion-in-memory [entity])))](t2/delete! :model/Dimension :field_id (:id email))(load! (assoc field :dimensions [dimension]))(is (= "Email" (t2/select-one-fn :name :model/Dimension :field_id (:id email))))@@ -348,7 +332,7 @@:fields [(ts/extract-one "FieldUserSettings" (:id age))]}](t2/delete! :model/FieldUserSettings :field_id (:id age))(t2/insert! :model/FieldUserSettings {:field_id (:id email) :description "stale"})- (serdes.load/load-metabase! (ingestion-in-memory [legacy]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [legacy]))(is (= "edited" (t2/select-one-fn :description :model/FieldUserSettings :field_id (:id age))))(is (not (t2/exists? :model/FieldUserSettings :field_id (:id email))))))))@@ -361,7 +345,7 @@_ (t2/insert! :model/FieldUserSettings {:field_id (:id email) :description "edited"})settings (ts/extract-one "FieldUserSettings" (:id email))](t2/update! :model/FieldUserSettings (:id email) {:display_name "Local"})- (serdes.load/load-metabase! (ingestion-in-memory [settings]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [settings]))(is (=? {:description "edited" :display_name nil}(t2/select-one :model/FieldUserSettings :field_id (:id email))))))))@@ -376,7 +360,7 @@extracted (ts/extract-one "Dimension" (:entity_id dimension))tables (t2/count :model/Table)](t2/delete! :model/Dimension :field_id (:id tier))- (serdes.load/load-metabase! (ingestion-in-memory [extracted]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [extracted]))(is (= [(:id tier)] (t2/select-fn-vec :field_id :model/Dimension :entity_id (:entity_id dimension))))(is (= tables (t2/count :model/Table)))))))@@ -423,7 +407,7 @@;; rather than aborting, and everything else lands. We do NOT promise a strict (default) import works.(testing "import under continue-on-error doesn't crash; the dangling dashboard is skipped, the rest lands"(ts/with-db dest-db- (let [report (serdes.load/load-metabase! (ingestion-in-memory @serialized) {:continue-on-error true})]+ (let [report (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized) {:continue-on-error true})](is (some? (t2/select-one :model/Collection :entity_id @coll-eid)) "target collection imported")(is (some? (t2/select-one :model/Card :entity_id @clean-eid)) "clean card imported")(is (nil? (t2/select-one :model/Card :entity_id @escaped-eid)) "escaped card not imported")@@ -452,7 +436,7 @@(ts/with-db dest-db(ts/create! :model/Collection :name "Unrelated Collection")(ts/create! :model/Collection :name "Parent Collection" :location "/" :entity_id (:entity_id @parent))- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [parent-dest (t2/select-one :model/Collection :entity_id (:entity_id @parent))child-dest (t2/select-one :model/Collection :entity_id (:entity_id @child))grandchild-dest (t2/select-one :model/Collection :entity_id (:entity_id @grandchild))]@@ -514,7 +498,7 @@:parent_id))))(testing "deserialization works properly, keeping the same-named tables apart"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(reset! db1d (t2/select-one :model/Database :name (:name @db1s)))(reset! db2d (t2/select-one :model/Database :name (:name @db2s)))(is (= 3 (t2/count :model/Database)))@@ -589,7 +573,7 @@(reset! field2d (ts/create! :model/Field :name "subtotal" :table_id (:id @table2d)))(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"));; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(reset! db1d (t2/select-one :model/Database :name "my-db"))(reset! table1d (t2/select-one :model/Table :name "customers"))@@ -652,7 +636,7 @@field (ts/create! :model/Field :name "category" :table_id (:id table):base_type :type/Text :semantic_type :type/Category:has_field_values :list)]- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [dest-card (t2/select-one :model/Card :name "Products Model")cat (first (filter #(= "category" (:name %)) (:result_metadata dest-card)))](is (number? (:id cat))@@ -694,7 +678,7 @@(testing "deserializing synthesizes inactive Table and Field for the dangling references"(ts/with-db dest-db(ts/create! :model/Database :name "my-db")- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [db (t2/select-one :model/Database :name "my-db")table (t2/select-one :model/Table :name "customers" :db_id (:id db))field (and table (t2/select-one :model/Field :name "age" :table_id (:id table)))@@ -756,7 +740,7 @@(reset! field2d (ts/create! :model/Field :name "subtotal" :table_id (:id @table2d)))(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"));; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(reset! db1d (t2/select-one :model/Database :name "my-db"))(reset! table1d (t2/select-one :model/Table :name "customers"))@@ -840,7 +824,7 @@_ (ts/create! :model/Field :name "subtotal" :table_id (:id table2d))_ (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")];; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(let [db1d (t2/select-one :model/Database :name "my-db")table1d (t2/select-one :model/Table :name "sales")@@ -904,7 +888,7 @@(ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]");; Load the serialized content- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(let [db1d (t2/select-one :model/Database :name "my-db")table1d (t2/select-one :model/Table :name "sales")@@ -967,7 +951,7 @@(ts/with-db dest-db(reset! user1s (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"));; Load the serialized content- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(reset! db1d (t2/select-one :model/Database :name "my-db"))(reset! table1d (t2/select-one :model/Table :name "products"))@@ -1182,7 +1166,7 @@(ts/create! :model/Field :name "address" :table_id (:id @table2d))(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"));; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(reset! db1d (t2/select-one :model/Database :name "my-db"))(reset! table1d (t2/select-one :model/Table :name "orders"))@@ -1302,7 +1286,7 @@:creator_id (:id @user1d):timezone "America/New_York"));; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(reset! timeline2d (t2/select-one :model/Timeline :entity_id (:entity_id @timeline2s)))(reset! eventsT1 (t2/select :model/TimelineEvent :timeline_id (:id @timeline1d)))@@ -1344,7 +1328,7 @@:timestamp "2027-04-20T00:00:00Z":time_matters false:timezone "UTC"}]}]- (serdes.load/load-metabase! (ingestion-in-memory [timeline]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [timeline]))(let [timeline-id (t2/select-one-pk :model/Timeline :entity_id timeline-eid)[event] (t2/select :model/TimelineEvent :timeline_id timeline-id)](is (= 1 (t2/count :model/TimelineEvent :timeline_id timeline-id)))@@ -1386,7 +1370,7 @@(ts/create! :model/Dashboard :name "Other dashboard C")(reset! user1d (ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]"));; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(reset! dash1d (t2/select-one :model/Dashboard :name "My Dashboard"))(reset! dash2d (t2/select-one :model/Dashboard :name "Linked dashboard"))(testing "the Dashboards and Users have different IDs now"@@ -1475,7 +1459,7 @@;; The :values are different here; they should get overwritten by the update.(reset! fv1d (ts/create! :model/FieldValues :field_id (:id @field1d) :values ["WA" "NC" "NM" "WI"]));; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(reset! fv1d (t2/select-one :model/FieldValues :field_id (:id @field1d)))(reset! fv2d (t2/select-one :model/FieldValues :field_id (:id @field2d)))@@ -1527,7 +1511,7 @@(testing "deserializing restores collection_id correctly"(ts/with-db dest-db;; Load the serialized content- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Fetch the relevant bits(reset! db1d (t2/select-one :model/Database :name "my-db"))(reset! coll1d (t2/select-one :model/Collection :name "Publishing Collection"))@@ -1550,7 +1534,7 @@collection-eid (u/generate-nano-id)card-path [{:model "Card" :id card-eid}]timeline-path [{:model "Timeline" :id timeline-eid}]- archive (ingestion-in-memory+ archive (ts/ingestion-in-memory[{:serdes/meta card-path:entity_id card-eid:name "Question with events"@@ -1591,32 +1575,32 @@(ts/create! :model/Field :name "STATE" :table_id (:id @table1s))(ts/create! :model/User :first_name "Geddy" :last_name "Lee" :email "[email protected]")(testing "depending on existing values works"- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]- :created_at (t/instant)- :creator_id "[email protected]"- :database_id "my-db"- :dataset_query {:database "my-db"- :type :query- :query {:source-table ["my-db" nil "CUSTOMERS"]}}- :display :table- :entity_id "0123456789abcdef_0123"- :name "Some card"- :table_id ["my-db" nil "CUSTOMERS"]- :visualization_settings {}}])]+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]+ :created_at (t/instant)+ :creator_id "[email protected]"+ :database_id "my-db"+ :dataset_query {:database "my-db"+ :type :query+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}+ :display :table+ :entity_id "0123456789abcdef_0123"+ :name "Some card"+ :table_id ["my-db" nil "CUSTOMERS"]+ :visualization_settings {}}])](is (some? (serdes.load/load-metabase! ingestion)))))(testing "depending on a nonexisting database creates a stub database"- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]- :created_at (t/instant)- :creator_id "[email protected]"- :database_id "bad-db"- :dataset_query {:database "bad-db"- :type :query- :query {:source-table ["bad-db" nil "CUSTOMERS"]}}- :display :table- :entity_id "0123456789abcdef_0123"- :name "Some card"- :table_id ["bad-db" nil "CUSTOMERS"]- :visualization_settings {}}])+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0123"}]+ :created_at (t/instant)+ :creator_id "[email protected]"+ :database_id "bad-db"+ :dataset_query {:database "bad-db"+ :type :query+ :query {:source-table ["bad-db" nil "CUSTOMERS"]}}+ :display :table+ :entity_id "0123456789abcdef_0123"+ :name "Some card"+ :table_id ["bad-db" nil "CUSTOMERS"]+ :visualization_settings {}}])_ (serdes.load/load-metabase! ingestion)stub (t2/select-one :model/Database :name "bad-db")](is (=? {:engine :postgres@@ -1629,19 +1613,19 @@(is (= (:id stub)(t2/select-one-fn :database_id :model/Card :entity_id "0123456789abcdef_0123")))))(testing "depending on nonexisting values fails"- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0124"}]- :collection_id "missingcollection0001"- :created_at (t/instant)- :creator_id "[email protected]"- :database_id "my-db"- :dataset_query {:database "my-db"- :type :query- :query {:source-table ["my-db" nil "CUSTOMERS"]}}- :display :table- :entity_id "0123456789abcdef_0124"- :name "Other card"- :table_id ["my-db" nil "CUSTOMERS"]- :visualization_settings {}}])+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0124"}]+ :collection_id "missingcollection0001"+ :created_at (t/instant)+ :creator_id "[email protected]"+ :database_id "my-db"+ :dataset_query {:database "my-db"+ :type :query+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}+ :display :table+ :entity_id "0123456789abcdef_0124"+ :name "Other card"+ :table_id ["my-db" nil "CUSTOMERS"]+ :visualization_settings {}}])e (try(serdes.load/load-metabase! ingestion)nil@@ -1658,21 +1642,21 @@(testing "attaching the referrer does not duplicate the error through the cause chain"(is (nil? (ex-cause e))))))(testing "a result_metadata table_id naming an absent database creates a stub database"- (let [ingestion (ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0125"}]- :created_at (t/instant)- :creator_id "[email protected]"- :database_id "my-db"- :dataset_query {:database "my-db"- :type :query- :query {:source-table ["my-db" nil "CUSTOMERS"]}}- :display :table- :entity_id "0123456789abcdef_0125"- :name "Some card"- :result_metadata [{:name "STATE"- :base_type :type/Text- :table_id ["absent-db" nil "CUSTOMERS"]}]- :table_id ["my-db" nil "CUSTOMERS"]- :visualization_settings {}}])]+ (let [ingestion (ts/ingestion-in-memory [{:serdes/meta [{:model "Card" :id "0123456789abcdef_0125"}]+ :created_at (t/instant)+ :creator_id "[email protected]"+ :database_id "my-db"+ :dataset_query {:database "my-db"+ :type :query+ :query {:source-table ["my-db" nil "CUSTOMERS"]}}+ :display :table+ :entity_id "0123456789abcdef_0125"+ :name "Some card"+ :result_metadata [{:name "STATE"+ :base_type :type/Text+ :table_id ["absent-db" nil "CUSTOMERS"]}]+ :table_id ["my-db" nil "CUSTOMERS"]+ :visualization_settings {}}])](serdes.load/load-metabase! ingestion)(is (true? (t2/select-one-fn :is_stub :model/Database :name "absent-db")))))))))@@ -1707,7 +1691,7 @@(:dataset_query @extracted))))(testing "when loading"(let [new-eid (u/generate-nano-id)- ingestion (ingestion-in-memory [(assoc @extracted :entity_id new-eid)])]+ ingestion (ts/ingestion-in-memory [(assoc @extracted :entity_id new-eid)])](is (some? (serdes.load/load-metabase! ingestion)))(is (= (:id @snippet1s)(-> (t2/select-one :model/Card :entity_id new-eid)@@ -1724,7 +1708,7 @@id1 (u/generate-nano-id)id2 (u/generate-nano-id)load! #(serdes.load/load-metabase!- (ingestion-in-memory [(serdes/extract-one "NativeQuerySnippet" {} %)]))]+ (ts/ingestion-in-memory [(serdes/extract-one "NativeQuerySnippet" {} %)]))](testing "setup is correct"(is (= (:entity_id snippet)(t2/select-one-fn :entity_id :model/NativeQuerySnippet :name unique-name))))@@ -1759,7 +1743,7 @@(:template_tags (first @serialized))))))(ts/with-db dest-db(ts/create! :model/NativeQuerySnippet :name "Unrelated" :content "3 = 3")- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (=? {"snippet: A" {:snippet-id (t2/select-one-pk :model/NativeQuerySnippet :name "A")}}(t2/select-one-fn :template_tags :model/NativeQuerySnippet :name "B"))))))))@@ -1774,7 +1758,7 @@:content "WHERE id = {{id}}":creator_id "[email protected]":entity_id "test-entity-1"}- ingestion (ingestion-in-memory [snippet-data])]+ ingestion (ts/ingestion-in-memory [snippet-data])](serdes.load/load-metabase! ingestion)(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-1")];; Toucan hooks compute the template-tags:@@ -1792,7 +1776,7 @@:template_tags {}:creator_id "[email protected]":entity_id "test-entity-2"}- ingestion (ingestion-in-memory [snippet-data])]+ ingestion (ts/ingestion-in-memory [snippet-data])](serdes.load/load-metabase! ingestion)(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-2")](is (= {} template-tags))))))@@ -1810,7 +1794,7 @@:display-name "Snippet: WOOP"}}:creator_id "[email protected]":entity_id "test-entity-3"}- ingestion (ingestion-in-memory [snippet-data])]+ ingestion (ts/ingestion-in-memory [snippet-data])](serdes.load/load-metabase! ingestion)(let [template-tags (t2/select-one-fn :template_tags :model/NativeQuerySnippet :entity_id "test-entity-3")](is (=? {"snippet: id" {:type :snippet@@ -1874,7 +1858,7 @@:dataset_query {:type :query:database (:id db):query {:source-table (:id table)}})))- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [target-id (t2/select-one-fn :id :model/Card :entity_id @target-eid)new-tag-name (str "#" target-id "-bh-population-model")stored-query #(t2/select-one-fn :dataset_query :model/Card :entity_id @native-eid)@@ -1890,7 +1874,7 @@:card-id target-id}](-> query :stages first :template-tags)))(testing "re-importing over the existing card (the update path) repairs the same way"- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (= query (stored-query))))))))))(deftest load-action-test@@ -1924,7 +1908,7 @@(is (string? (:type action-serialized))))))))(testing "loading succeeds"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [action (action/select-action :entity_id eid)](is (some? action))(is (some? (:dataset_query action)))@@ -1953,7 +1937,7 @@(reset! dashcard2d (ts/create! :model/DashboardCard :dashboard_id (:id @dash1d)))(reset! tab2d (ts/create! :model/DashboardTab :name "Tab 2" :dashboard_id (:id @dash1d)));; Load the serialized content.- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(reset! dash1d (-> (t2/select-one :model/Dashboard :name "My Dashboard")(t2/hydrate :dashcards)(t2/hydrate :tabs)))@@ -1988,7 +1972,7 @@series3s (ts/create! :model/DashboardCardSeries :dashboardcard_id (:id dashcard1s) :card_id (:id series-card3s) :position 2)extract1 (into [] (serdes.extract/extract {:no-settings true}))](ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory extract1))+ (serdes.load/load-metabase! (ts/ingestion-in-memory extract1))(ts/with-db source-db;; delete the 1st series and update the 3rd series to have position 0, and the 2nd series to have position 1(t2/delete! :model/DashboardCardSeries (:id series1s))@@ -2003,7 +1987,7 @@(testing "Sense check: there are 3 series for the dashboard card initially"(is (= 3(t2/count :model/DashboardCardSeries :dashboardcard_id (:dashboardcard_id series-to-be-deleted)))))- (serdes.load/load-metabase! (ingestion-in-memory extract2))+ (serdes.load/load-metabase! (ts/ingestion-in-memory extract2))(let [dash1d (-> (t2/select-one :model/Dashboard :name "My Dashboard")(t2/hydrate [:dashcards :series]))](testing "Dashboard cards have the same entity ID"@@ -2035,7 +2019,7 @@:model/DashboardCardSeries _s2 {:dashboardcard_id (:id dc2) :card_id (:id sc) :position 0}](let [extract (into [] (serdes.extract/extract {:no-settings true}))](ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory extract))+ (serdes.load/load-metabase! (ts/ingestion-in-memory extract))(testing "Both series get imported even though they point at the same card"(is (= 2(t2/count :model/DashboardCardSeries)))))))))))@@ -2072,7 +2056,7 @@(into []))))))(testing "The extraneous keys do not interfere with loading"(ts/with-db dest-db- (is (serdes.load/load-metabase! (ingestion-in-memory @serialized))))))))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))))))))(deftest tx-test(mt/with-empty-h2-app-db!@@ -2085,7 +2069,7 @@vec))(testing "Load completes successfully"(t2/update! :model/Card {:id (:id card)} {:name (str "qwe_" (:name card))})- (is (serdes.load/load-metabase! (ingestion-in-memory @serialized)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized)))(is (= (:name card)(t2/select-one-fn :name :model/Card :id (:id card)))))(testing "Partial load commits successful entities; failed entity does not persist"@@ -2098,7 +2082,7 @@(throw (ex-info "oops, error" {}))(load-update! model adjusted local)))](is (thrown? clojure.lang.ExceptionInfo- (serdes.load/load-metabase! (ingestion-in-memory @serialized))))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))));; Collection loaded successfully in its own transaction — committed despite Card failure(is (= (:name coll)(t2/select-one-fn :name :model/Collection :id (:id coll))))@@ -2133,7 +2117,7 @@"40001" ; SQL state40001))))) ; H2 error code(load-update! model adjusted local))]- (is (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))"Import should succeed after retrying the deadlocked entity")(is (= (:name card)(t2/select-one-fn :name :model/Card :id (:id card)))@@ -2149,7 +2133,7 @@(throw (ex-info "constraint violation" {})))(load-update! model adjusted local))](is (thrown? clojure.lang.ExceptionInfo- (serdes.load/load-metabase! (ingestion-in-memory @serialized)))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized)))"Non-transient error should propagate")(is (= 1 @call-count)"Should not retry non-transient errors"))))@@ -2163,7 +2147,7 @@(throw (ex-info "oops" {}))(load-update! model adjusted local)))](is (thrown? clojure.lang.ExceptionInfo- (serdes.load/load-metabase! (ingestion-in-memory @serialized))))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))));; With per-entity transactions, the Collection commit survives the Card failure.;; On master (single transaction), the Collection update is rolled back.(is (= (:name coll)@@ -2217,7 +2201,7 @@select-target #(-> % :visualization_settings :click_behavior :targetId)];; loaded to the same db so we know ids won't be corrupted(testing "Dashcard ids won't be changed by loading on top of them"- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(is (= (:id dc1)(t2/select-one-fn :id :model/DashboardCard :entity_id (:entity_id dc1))))(is (= (:id dc2)@@ -2233,7 +2217,7 @@;; loaded to a different db so we know it works for sure instead of relying on data being present in the;; database(ts/with-db dest-db- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(testing "Circular dependencies work for Dashboard Questions as well"(is (= (t2/select-one-fn :id :model/Dashboard :entity_id (:entity_id dash1))(t2/select-one-fn :dashboard_id :model/Card :entity_id (:entity_id card-2))))))))))@@ -2259,13 +2243,13 @@ser (into [] (serdes.extract/extract {:no-settings true:no-data-model false}))](testing "loading on top of the existing card"- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(is (= (:id card)(-> (t2/select-one :model/Card :entity_id (:entity_id card)):parameters first :values_source_config :card_id))))(testing "loading into an empty database"(ts/with-db dest-db- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(let [new-card (t2/select-one :model/Card :entity_id (:entity_id card))](is (= (:id new-card)(-> new-card :parameters first :values_source_config :card_id)))))))))))@@ -2282,7 +2266,7 @@ser (into [] (serdes.extract/extract {:no-settings true:no-data-model false}))](testing "loading on top of the existing cards"- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(is (= (:id card-b)(-> (t2/select-one :model/Card :entity_id (:entity_id card-a)):parameters first :values_source_config :card_id)))@@ -2291,7 +2275,7 @@:parameters first :values_source_config :card_id))))(testing "loading into an empty database"(ts/with-db dest-db- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(let [new-a (t2/select-one :model/Card :entity_id (:entity_id card-a))new-b (t2/select-one :model/Card :entity_id (:entity_id card-b))](is (= (:id new-b)@@ -2312,13 +2296,13 @@ser (into [] (serdes.extract/extract {:no-settings true:no-data-model false}))](testing "loading on top of the existing dashboard"- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(is (= (:id card)(-> (t2/select-one :model/Dashboard :entity_id (:entity_id dash)):parameters first :values_source_config :card_id))))(testing "loading into an empty database"(ts/with-db dest-db- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(let [new-dash (t2/select-one :model/Dashboard :entity_id (:entity_id dash))new-card (t2/select-one :model/Card :entity_id (:entity_id card))](is (= (:id new-card)@@ -2345,19 +2329,19 @@messages (fn [e] (map ex-message (take-while some? (iterate ex-cause e))))](ts/with-db dest-db(testing "A dashboard question loads into Library Dashboards before its dashboard"- (is (serdes.load/load-metabase! (ingestion-in-memory (cons (card-ser dq) (remove #{(card-ser standalone)} ser)))))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory (cons (card-ser dq) (remove #{(card-ser standalone)} ser)))))(is (= (t2/select-one-pk :model/Dashboard :entity_id (:entity_id dash))(t2/select-one-fn :dashboard_id :model/Card :entity_id (:entity_id dq)))))(testing "A standalone question can't load into Library Dashboards"(let [e (is (thrown? clojure.lang.ExceptionInfo(serdes.load/load-metabase!- (ingestion-in-memory [(assoc (card-ser standalone) :collection_id (:entity_id coll))]))))]+ (ts/ingestion-in-memory [(assoc (card-ser standalone) :collection_id (:entity_id coll))]))))](is (some #(re-find #"Can only add dashboards to the 'Dashboards' collection" %) (messages e)))(is (not (t2/exists? :model/Card :entity_id (:entity_id standalone))))))(testing "A dashboard question can't become a standalone question in Library Dashboards"(let [e (is (thrown? clojure.lang.ExceptionInfo(serdes.load/load-metabase!- (ingestion-in-memory [(assoc (card-ser dq) :dashboard_id nil)]))))]+ (ts/ingestion-in-memory [(assoc (card-ser dq) :dashboard_id nil)]))))](is (some #(re-find #"Can only add dashboards to the 'Dashboards' collection" %) (messages e)))))))))))(deftest continue-on-error-test@@ -2396,7 +2380,7 @@:no-transforms true}))changed (change-ser ser {(:entity_id c2) {:collection_id "does-not-exist"}})](mt/with-log-messages-for-level [messages [metabase-enterprise :warn]]- (let [report (serdes.load/load-metabase! (ingestion-in-memory changed) {:continue-on-error true})]+ (let [report (serdes.load/load-metabase! (ts/ingestion-in-memory changed) {:continue-on-error true})](is (= 1 (count (:errors report))))(is (= 3 (count (:seen report)))))(let [log-msgs (logs-extract #"Skipping deserialization error: (.*)" (messages))]@@ -2426,12 +2410,12 @@(is (not-any? :details dbs))(ts/with-db dest-db(testing "loading still works even if there are no details"- (serdes.load/load-metabase! (ingestion-in-memory extracted))+ (serdes.load/load-metabase! (ts/ingestion-in-memory extracted))(is (= {}(t2/select-one-fn :details :model/Database)))(testing "If we did not export details - it won't override existing data"(t2/update! :model/Database {:details {:other "secret"}})- (serdes.load/load-metabase! (ingestion-in-memory extracted))+ (serdes.load/load-metabase! (ts/ingestion-in-memory extracted))(is (= {:other "secret"}(t2/select-one-fn :details :model/Database)))))))))(mt/with-temp [:model/Database _ {:name "My Database"@@ -2453,7 +2437,7 @@(mt/with-temp [:model/Dimension d2 {:name "Absolutely Other Dimension":field_id (mt/id :venues :price):type "internal"}]- (serdes.load/load-metabase! (ingestion-in-memory ser))+ (serdes.load/load-metabase! (ts/ingestion-in-memory ser))(is (= (:entity_id d1)(t2/select-one-fn :entity_id :model/Dimension :field_id (mt/id :venues :price))))(is (= nil@@ -2484,7 +2468,7 @@{:model "Field" :id "field"}]}(ts/extract-one "Field" (:id f3))))(t2/update! :model/Field (:id f3) {:description "some new one"})- (is (serdes.load/load-metabase! (ingestion-in-memory ser)))+ (is (serdes.load/load-metabase! (ts/ingestion-in-memory ser)))(is (= "desc"(t2/select-one-fn :description :model/Field (:id f3)))))))@@ -2513,7 +2497,7 @@(testing "the unlabeled Field exports no data_sensitivity key"(is (not (contains? (field-ser @serialized "CONTACT_ROW") :data_sensitivity))))))(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [db-id (t2/select-one-fn :id :model/Database :name "labeled-db")table-id (t2/select-one-fn :id :model/Table :name "CONTACTS" :db_id db-id)email-id (t2/select-one-fn :id :model/Field :name "CONTACT_EMAIL" :table_id table-id)@@ -2532,12 +2516,12 @@[coll-ser] (serdes.extract/extract {:targets [["Collection" (:id coll)]]})new-coll (assoc coll-ser :entity_id nil)coll-count (fn [] (t2/count :model/Collection :name "mycoll"))]- (serdes.load/load-metabase! (ingestion-in-memory [new-coll]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [new-coll]))(is (= 2 (coll-count)))- (serdes.load/load-metabase! (ingestion-in-memory [new-coll]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [new-coll]))(is (= 3 (coll-count)))(testing "absent :entity_id also works"- (serdes.load/load-metabase! (ingestion-in-memory [(dissoc coll-ser :entity_id)]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(dissoc coll-ser :entity_id)]))(is (= 4 (coll-count)))))))(deftest import-published-table-with-existing-database-test@@ -2571,7 +2555,7 @@;; Pre-create the database on target(let [target-db (ts/create! :model/Database :name "shared-db")];; Load the serialized content- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized));; Verify the table was imported correctly(let [imported-table (t2/select-one :model/Table :name "published_table")imported-coll (t2/select-one :model/Collection :entity_id @coll-eid)]@@ -2604,7 +2588,7 @@:no-settings true}))))))(testing "import creates a stub database"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [stub (t2/select-one :model/Database :name "source-only-db")](is (=? {:engine :postgres:details {}@@ -2625,7 +2609,7 @@(is (not (contains? @serialized :is_stub)))))(ts/with-db dest-db(let [real (ts/create! :model/Database :name "Prod" :engine :postgres :details {:host "prod"})]- (serdes.load/load-metabase! (ingestion-in-memory [(assoc @serialized :is_stub true)]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(assoc @serialized :is_stub true)]))(is (=? {:details {:host "prod"}:is_stub false}(t2/select-one :model/Database :id (:id real))))))))))@@ -2650,7 +2634,7 @@@serialized)](ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [segment (t2/select-one :model/Segment :name "Minors")table (t2/select-one :model/Table :name "customers")](is (some? segment))@@ -2676,7 +2660,7 @@@serialized)](ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [measure (t2/select-one :model/Measure :name "Total Sales")table (t2/select-one :model/Table :name "sales")](is (some? measure))@@ -2705,7 +2689,7 @@@serialized)](ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [card (t2/select-one :model/Card :name "Example Card")db (t2/select-one :model/Database :name "my-db")table (t2/select-one :model/Table :name "customers")]@@ -2744,7 +2728,7 @@@serialized)](ts/with-db dest-db(t2/delete! :model/TransformTag)- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [transform (t2/select-one :model/Transform :name "Test Transform")db (t2/select-one :model/Database :name "my-db")](is (some? transform))@@ -2784,7 +2768,7 @@(let [db (ts/create! :model/Database :name "padding-db")table (ts/create! :model/Table :name "padding" :db_id (:id db))](ts/create! :model/Field :name "padding_field" :table_id (:id table)))- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [transform (t2/select-one :model/Transform :name "Checkpoint Transform")field (t2/select-one :model/Field :name "updated_at")](is (some? transform))@@ -2823,7 +2807,7 @@(get-in transform [:source :source-incremental-strategy :checkpoint-filter-field-id])))))(ts/with-db dest-db(t2/delete! :model/TransformTag)- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (= 999999999(get-in (t2/select-one :model/Transform :name "Stale Checkpoint Transform")[:source :source-incremental-strategy :checkpoint-filter-field-id]))@@ -2867,7 +2851,7 @@(is (some? (get-in extracted [:source :query])))))(ts/with-db dest-db(t2/delete! :model/TransformTag)- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [transform (t2/select-one :model/Transform :name "Orphan Transform")](testing "transform round-trips and ends up as a tombstone in the destination instance"(is (some? transform))@@ -2904,7 +2888,7 @@(reset! serialized (into [] (serdes.extract/extract {})))))(ts/with-db dest-db(t2/delete! :model/TransformTag)- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [transform (t2/select-one :model/Transform :name "Hello Transform")table (t2/select-one :model/Table :name "hello_transforms_world")](is (some? transform))@@ -2924,7 +2908,7 @@entity))@serialized)](ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")](is (some? dashboard)))))))))@@ -2949,7 +2933,7 @@entity)))@serialized)](ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))](is (some? dashboard))@@ -2986,7 +2970,7 @@entity)))@serialized)](ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [dashboard (t2/select-one :model/Dashboard :name "Test Dashboard")dashcards (t2/select :model/DashboardCard :dashboard_id (:id dashboard))series (t2/select :model/DashboardCardSeries :dashboardcard_id (:id (first dashcards)))]@@ -3010,7 +2994,7 @@@serialized))))(testing "loading into an empty database succeeds"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [channels (t2/select :model/Channel :name "Test Email Channel")](is (= 1 (count channels)))(is (= "Test Email Channel" (:name (first channels))))@@ -3018,7 +3002,7 @@(is (= "A test email channel" (:description (first channels)))))))(testing "loading again does not duplicate"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (= 1 (t2/count :model/Channel :name "Test Email Channel")))))))))(deftest metabot-roundtrip-test@@ -3038,14 +3022,14 @@@serialized))))(testing "loading into an empty database succeeds"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [metabots (t2/select :model/Metabot :entity_id @metabot-eid)](is (= 1 (count metabots)))(is (= "Test Bot" (:name (first metabots))))(is (= "A test metabot" (:description (first metabots)))))))(testing "loading again does not duplicate"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (= 1 (t2/count :model/Metabot :entity_id @metabot-eid)))))))))(deftest channel-minimal-required-properties-test@@ -3064,7 +3048,7 @@entity))@serialized)](ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [channel (t2/select-one :model/Channel :name "Minimal Channel")](is (some? channel))(is (= :channel/email (:type channel))))))))))@@ -3084,7 +3068,7 @@entity))@serialized)](ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory minimal))+ (serdes.load/load-metabase! (ts/ingestion-in-memory minimal))(let [metabot (t2/select-one :model/Metabot :name "Minimal Bot")](is (some? metabot))(is (= "Minimal Bot" (:name metabot))))))))))@@ -3120,7 +3104,7 @@(is (not (contains? card-ser :query_type)) "query_type should be stripped from export"))))(ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [card (t2/select-one :model/Card :name "Round Trip Card")db (t2/select-one :model/Database :name "my-db")table (t2/select-one :model/Table :name "orders")]@@ -3147,7 +3131,7 @@(is (not (contains? seg-ser :table_id)) "table_id should be stripped from export"))))(ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [segment (t2/select-one :model/Segment :name "Minors")table (t2/select-one :model/Table :name "customers")](is (some? segment))@@ -3170,7 +3154,7 @@(is (not (contains? msr-ser :table_id)) "table_id should be stripped from export"))))(ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [measure (t2/select-one :model/Measure :name "Total Sales")table (t2/select-one :model/Table :name "sales")](is (some? measure))@@ -3202,7 +3186,7 @@(is (contains? card-ser :database_id) "database_id exported — not derivable from empty query"))))(ts/with-db dest-db(ts/create! :model/User :first_name "Tom" :last_name "Scholz" :email "[email protected]")- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [card (t2/select-one :model/Card :name "Empty Query Card")db (t2/select-one :model/Database :name "my-db")](is (some? card))@@ -3265,7 +3249,7 @@@serialized))))(testing "loading into destination preserves library structure"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [lib-dest (t2/select-one :model/Collection :entity_id (:entity_id @library))data-dest (t2/select-one :model/Collection :entity_id (:entity_id @data-root))metrics-dest (t2/select-one :model/Collection :entity_id (:entity_id @metrics-root))@@ -3325,7 +3309,7 @@:object [:like (format "/collection/%d/%%" (:id data-dest))])))](testing "custom permissions exist before import"(is (seq perms-before)))- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(let [data-after (t2/select-one :model/Collection :entity_id eid-data)perms-after (set (map #(select-keys % [:group_id :object])(t2/select :model/Permissions@@ -3347,12 +3331,12 @@(is (contains? (ids-by-model @serialized "Glossary") @eid)))(testing "importing into an empty app db reproduces the term, definition and entity_id"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (=? [{:term "ARR" :definition "Annual recurring revenue" :entity_id @eid}](t2/select :model/Glossary)))))(testing "importing again updates in place rather than duplicating"(ts/with-db dest-db- (serdes.load/load-metabase! (ingestion-in-memory @serialized))+ (serdes.load/load-metabase! (ts/ingestion-in-memory @serialized))(is (= 1 (t2/count :model/Glossary))))))))(deftest glossary-load-find-local-test@@ -3379,19 +3363,19 @@(testing "a term-keyed file exported before entity_id existed updates the same-term row and keeps its entity_id"(mt/with-empty-h2-app-db!(let [{local-eid :entity_id} (ts/create! :model/Glossary :term "ARR" :definition "local")]- (serdes.load/load-metabase! (ingestion-in-memory [(glossary-file "ARR" nil)]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(glossary-file "ARR" nil)]))(is (=? [{:term "ARR" :definition "Annual recurring revenue (imported)" :entity_id local-eid}](t2/select :model/Glossary))))))(testing "a file whose term exists locally under another entity_id updates that row in place and adopts the file's entity_id"(mt/with-empty-h2-app-db!(let [file-eid "glossaryfileeid000001"{local-id :id} (ts/create! :model/Glossary :term "ARR" :definition "local")]- (serdes.load/load-metabase! (ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))(is (=? [{:id local-id :term "ARR" :definition "Annual recurring revenue (imported)" :entity_id file-eid}](t2/select :model/Glossary))))))(testing "a file whose entity_id and term are both new inserts a row"(mt/with-empty-h2-app-db!(let [file-eid "glossaryfileeid000002"]- (serdes.load/load-metabase! (ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))+ (serdes.load/load-metabase! (ts/ingestion-in-memory [(glossary-file file-eid {:entity_id file-eid})]))(is (=? [{:term "ARR" :entity_id file-eid}](t2/select :model/Glossary))))))))diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex 290fb3a5278..cc9e83d4427 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -151,6 +151,7 @@custom-viz-plugindashboardsdata-apps+ dependenciesdocumentsapi.eid-translationembeddingdiff --git a/src/metabase/batch_processing/impl.clj b/src/metabase/batch_processing/impl.cljindex 0c514721e41..17a48fed6e4 100644--- a/src/metabase/batch_processing/impl.clj+++ b/src/metabase/batch_processing/impl.clj@@ -109,6 +109,8 @@[:id :int]]sdk-tracked-rowid+timestamp+ [:map {:closed true}+ [:table-id ms/PositiveInt]][:map {:closed true}[:host :string][:timestamp (ms/InstanceOfClass java.time.temporal.Temporal)]]diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex c999db3b42a..1f81ab70485 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -49,7 +49,8 @@(derive :metabase/model)(derive :perms/use-parent-collection-perms)(derive :hook/timestamped?)- (derive :hook/entity-id))+ (derive :hook/entity-id)+ (derive :hook/dependencies))(defmethod mi/can-write? :model/Dashboard([instance]diff --git a/src/metabase/dashboards/models/dashboard_card.clj b/src/metabase/dashboards/models/dashboard_card.cljindex d4a38767129..582b0cab5c8 100644--- a/src/metabase/dashboards/models/dashboard_card.clj+++ b/src/metabase/dashboards/models/dashboard_card.clj@@ -24,6 +24,7 @@(derive ::mi/write-policy.full-perms-for-perms-set)(derive :hook/timestamped?)(derive :hook/entity-id)+ (derive :hook/dependencies);; Disabled for performance reasons, see update-dashboard-card!-call-count-test#_(derive :hook/search-index))diff --git a/src/metabase/dashboards/models/dashboard_card_series.clj b/src/metabase/dashboards/models/dashboard_card_series.cljindex a2baf27d580..1bf369bd121 100644--- a/src/metabase/dashboards/models/dashboard_card_series.clj+++ b/src/metabase/dashboards/models/dashboard_card_series.clj@@ -8,7 +8,8 @@(methodical/defmethod t2/table-name :model/DashboardCardSeries [_model] :dashboardcard_series)(doto :model/DashboardCardSeries- (derive :metabase/model))+ (derive :metabase/model)+ (derive :hook/dependencies));; Serializationdiff --git a/src/metabase/dashboards/models/dashboard_tab.clj b/src/metabase/dashboards/models/dashboard_tab.cljindex abefe6676ef..424f5f5bdee 100644--- a/src/metabase/dashboards/models/dashboard_tab.clj+++ b/src/metabase/dashboards/models/dashboard_tab.clj@@ -20,7 +20,8 @@(derive ::mi/read-policy.full-perms-for-perms-set)(derive ::mi/write-policy.full-perms-for-perms-set)(derive :hook/timestamped?)- (derive :hook/entity-id))+ (derive :hook/entity-id)+ (derive :hook/dependencies))(methodical/defmethod t2/model-for-automagic-hydration [:metabase.dashboards.models.dashboard-card/DashboardCard :dashboard_tab][_original-model _k]diff --git a/src/metabase/dependencies/hooks.clj b/src/metabase/dependencies/hooks.cljnew file mode 100644index 00000000000..c7504438bd7--- /dev/null+++ b/src/metabase/dependencies/hooks.clj@@ -0,0 +1,50 @@+(ns metabase.dependencies.hooks+ "Hooks that keep the dependency graph current as content is written, implemented by the enterprise module."+ (:require+ [metabase.premium-features.defenterprise :refer [defenterprise]]))++(defenterprise before-insert+ "Hook for the `model` row `instance` before it is inserted. The OSS implementation is a no-op."+ metabase-enterprise.dependencies.hooks+ [_model _instance]+ nil)++(defenterprise before-update+ "Hook for the `model` row `instance` before it is updated, whose Toucan changes say what changes. The OSS+ implementation is a no-op."+ metabase-enterprise.dependencies.hooks+ [_model _instance]+ nil)++(defenterprise before-delete+ "Hook for the `model` row `instance` before it is deleted. The OSS implementation is a no-op."+ metabase-enterprise.dependencies.hooks+ [_model _instance]+ nil)++(defenterprise after-database-sync+ "Hook for the Database with `database-id` after a sync. The OSS implementation is a no-op."+ metabase-enterprise.dependencies.hooks+ [_database-id]+ nil)++(defenterprise after-serdes-load+ "Hook for after a serdes load, during which the hooks mark what the load changes without triggering the jobs. The+ OSS implementation is a no-op."+ metabase-enterprise.dependencies.hooks+ []+ nil)++(defenterprise after-premium-token-update+ "Hook for after the premium token changed, which may have enabled the dependencies feature. The OSS implementation is+ a no-op."+ metabase-enterprise.dependencies.hooks+ []+ nil)++(defenterprise after-transform-run+ "Hook for the transform with `transform-id` after a run wrote its output Table with `table-id`, or nil when the+ table was not synced. The OSS implementation is a no-op."+ metabase-enterprise.dependencies.hooks+ [_transform-id _table-id]+ nil)diff --git a/src/metabase/documents/models/document.clj b/src/metabase/documents/models/document.cljindex 266ee63b7d2..d0e92040fce 100644--- a/src/metabase/documents/models/document.clj+++ b/src/metabase/documents/models/document.clj@@ -48,7 +48,8 @@(derive :metabase/model)(derive :perms/use-parent-collection-perms)(derive :hook/timestamped?)- (derive :hook/entity-id))+ (derive :hook/entity-id)+ (derive :hook/dependencies))(defonce ^{:doc "Predicate gating a document's *content* (not merely its existence) belowcollection-read, for documents whose rendered body embeds data the viewer may notdiff --git a/src/metabase/measures/models/measure.clj b/src/metabase/measures/models/measure.cljindex 318e30ecfb1..6e7153ee9b5 100644--- a/src/metabase/measures/models/measure.clj+++ b/src/metabase/measures/models/measure.clj@@ -56,7 +56,8 @@(doto :model/Measure(derive :metabase/model)(derive :hook/timestamped?)- (derive :hook/entity-id))+ (derive :hook/entity-id)+ (derive :hook/dependencies))(defmethod mi/can-read? :model/Measure([instance]diff --git a/src/metabase/models/interface.clj b/src/metabase/models/interface.cljindex ddb68b6f6c4..8c56d7a6f94 100644--- a/src/metabase/models/interface.clj+++ b/src/metabase/models/interface.clj@@ -18,6 +18,7 @@[malli.transform :as mtx][medley.core :as m][metabase.app-db.core :as mdb]+ [metabase.dependencies.hooks :as dependencies.hooks];; Toucan out-transforms normalize stored legacy MBQL on read; needed until the app db is MBQL 5^{:clj-kondo/ignore [:discouraged-namespace]} [metabase.legacy-mbql.normalize :as mbql.normalize];; stored card queries/refs are still legacy MBQL; validated against the legacy schema on read/write@@ -38,10 +39,14 @@[metabase.util.string :as string][methodical.core :as methodical][potemkin :as p]+ [toucan2.connection :as t2.conn][toucan2.core :as t2][toucan2.model :as t2.model]+ [toucan2.pipeline :as t2.pipeline][toucan2.protocols :as t2.protocols]+ [toucan2.tools.before-delete :as t2.before-delete][toucan2.tools.before-insert :as t2.before-insert]+ [toucan2.tools.before-update :as t2.before-update][toucan2.tools.hydrate :as t2.hydrate][toucan2.util :as t2.u])(:import@@ -676,6 +681,38 @@(methodical/prefer-method! #'t2.before-insert/before-insert :hook/timestamped? :hook/entity-id)(methodical/prefer-method! #'t2.before-insert/before-insert :hook/updated-at-timestamped? :hook/entity-id)(methodical/prefer-method! #'t2.before-insert/before-insert :hook/created-at-timestamped? :hook/entity-id)++(t2/define-before-insert :hook/dependencies+ [instance]+ (dependencies.hooks/before-insert &model instance)+ instance)++(t2/define-before-update :hook/dependencies+ [instance]+ (dependencies.hooks/before-update &model instance)+ instance)++(methodical/defmethod t2.pipeline/transduce-query :around [:toucan.query-type/update.* :hook/dependencies :default]+ "Run the before-update hooks of a `:hook/dependencies` model in the transaction of its UPDATE, which Toucan otherwise+ opens only after them, so dependency marks commit together with the UPDATE and after-commit work sees it."+ [rf query-type model parsed-args resolved-query]+ (t2.conn/with-transaction [_conn nil {:nested-transaction-rule :ignore}]+ (next-method rf query-type model parsed-args resolved-query)))++(t2/define-before-delete :hook/dependencies+ [instance]+ instance)++(methodical/defmethod t2.before-delete/before-delete :before :hook/dependencies+ [model instance]+ (dependencies.hooks/before-delete model instance)+ instance)++(doseq [hook [:hook/timestamped? :hook/created-at-timestamped? :hook/updated-at-timestamped? :hook/entity-id]]+ (methodical/prefer-method! #'t2.before-insert/before-insert hook :hook/dependencies))++(doseq [hook [:hook/timestamped? :hook/updated-at-timestamped?]]+ (methodical/prefer-method! #'t2.before-update/before-update hook :hook/dependencies));; --- helper fns(defn changes-with-pk"The row merged with the changes in pre-update hooks.diff --git a/src/metabase/models/serialization.clj b/src/metabase/models/serialization.cljindex 042e8a1ff56..ea852ffd30c 100644--- a/src/metabase/models/serialization.clj+++ b/src/metabase/models/serialization.clj@@ -1746,7 +1746,7 @@export-visualizer-settingsexport-pivot-tableexport-timeline-events- (update :column_settings export-column-settings))))+ (m/update-existing :column_settings export-column-settings))))(defn- import-viz-link-card[settings]@@ -1838,6 +1838,13 @@(m/update-existing :timeline.selected_timeline_ids import-fks entity-id? :model/Timeline)(m/update-existing :timeline.excluded_timeline_event_ids import-fks timeline-event-ref? :model/TimelineEvent)))+(defn- import-viz-column-settings+ "Import `:column_settings`, dropping the `nil` that older exports wrote for settings without any."+ [settings]+ (if (some? (:column_settings settings))+ (update settings :column_settings import-column-settings)+ (dissoc settings :column_settings)))+(defn import-visualization-settings"Given an EDN value as exported by [[export-visualization-settings]], convert its portable `[db schema table field]`references into Field IDs."@@ -1850,7 +1857,7 @@import-visualizer-settingsimport-pivot-tableimport-timeline-events- (update :column_settings import-column-settings))))+ import-viz-column-settings)))(defn- viz-link-card-deps[allow-int-ids? settings]diff --git a/src/metabase/native_query_snippets/models/native_query_snippet.clj b/src/metabase/native_query_snippets/models/native_query_snippet.cljindex 66d82484e4f..1d2e2255051 100644--- a/src/metabase/native_query_snippets/models/native_query_snippet.clj+++ b/src/metabase/native_query_snippets/models/native_query_snippet.clj@@ -25,7 +25,8 @@(doto :model/NativeQuerySnippet(derive :metabase/model)(derive :hook/timestamped?)- (derive :hook/entity-id))+ (derive :hook/entity-id)+ (derive :hook/dependencies));; TODO (Cam 2026-07-08) Change Native Query Snippets to store template tags as a list like we do in MBQL as of 63.(t2/deftransforms :model/NativeQuerySnippetdiff --git a/src/metabase/premium_features/token_check.clj b/src/metabase/premium_features/token_check.cljindex baf803453e3..4f2e2c52d3b 100644--- a/src/metabase/premium_features/token_check.clj+++ b/src/metabase/premium_features/token_check.clj@@ -18,7 +18,6 @@[metabase.analytics-interface.core :as analytics][metabase.app-db.core :as app-db][metabase.config.core :as config]- [metabase.events.core :as events][metabase.internal-stats.core :as internal-stats][metabase.premium-features.db :as premium-features.db][metabase.premium-features.defenterprise :refer [defenterprise]]@@ -614,8 +613,6 @@([checker token](-check-token checker token)))-(events/derive! :event/set-premium-embedding-token :metabase/event)-(defn -set-premium-embedding-token!"Setter for the [[metabase.premium-features.settings/token-status]] setting."[new-value]@@ -645,7 +642,7 @@:response/keys #{:error-details}}))))(log/info "Token is valid."))(setting/set-value-of-type! :string :premium-embedding-token new-value)- (events/publish-event! :event/set-premium-embedding-token {})+ ((requiring-resolve 'metabase.dependencies.hooks/after-premium-token-update))(catch Throwable e(log/errorf "Error setting premium features token: %s" (ex-message e));; merge in error-details if presentdiff --git a/src/metabase/queries/core.clj b/src/metabase/queries/core.cljindex 112ea8a0009..755b35237d9 100644--- a/src/metabase/queries/core.clj+++ b/src/metabase/queries/core.clj@@ -65,7 +65,6 @@update-card!visible-metric-cards-where-clause][metabase.queries.models.card.metadata- infer-metadatainfer-metadata-with-model-overridesmaybe-async-result-metadatarefresh-metadatadiff --git a/src/metabase/queries/models/card.clj b/src/metabase/queries/models/card.cljindex 439836d5269..882265f5fb2 100644--- a/src/metabase/queries/models/card.clj+++ b/src/metabase/queries/models/card.clj@@ -136,7 +136,8 @@;; You can read/write a Card if you can read/write its parent Collection(derive :perms/use-parent-collection-perms)(derive :hook/timestamped?)- (derive :hook/entity-id))+ (derive :hook/entity-id)+ (derive :hook/dependencies))(defn- parent-document-id"The `document_id` of `card`, or `::not-adjudicable` when the instance carries neither the columndiff --git a/src/metabase/segments/models/segment.clj b/src/metabase/segments/models/segment.cljindex 8f097a9bee2..d141cec702f 100644--- a/src/metabase/segments/models/segment.clj+++ b/src/metabase/segments/models/segment.clj@@ -78,7 +78,8 @@(doto :model/Segment(derive :metabase/model)(derive :hook/timestamped?)- (derive :hook/entity-id))+ (derive :hook/entity-id)+ (derive :hook/dependencies))(defmethod mi/can-read? :model/Segment([instance]diff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.cljindex 1eabacb2d7a..49b4f4cfd27 100644--- a/src/metabase/sync/sync_metadata.clj+++ b/src/metabase/sync/sync_metadata.clj@@ -8,6 +8,7 @@4. Sync FKs and indexes5. Sync Metabase Metadata table (`metabase.sync.sync-metadata.metabase-metadata`)"(:require+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.sync.fetch-metadata :as fetch-metadata][metabase.sync.interface :as i][metabase.sync.sync-metadata.dbms-version :as sync-dbms-ver]@@ -93,6 +94,7 @@(if abort-cause(sync-util/set-initial-database-sync-aborted! database abort-cause)(sync-util/set-initial-database-sync-complete! database))+ (dependencies.hooks/after-database-sync (:id database))results))(mu/defn sync-db-metadata!diff --git a/src/metabase/transforms/models/transform.clj b/src/metabase/transforms/models/transform.cljindex 27f63257cec..31f9ff7f845 100644--- a/src/metabase/transforms/models/transform.clj+++ b/src/metabase/transforms/models/transform.clj@@ -34,7 +34,7 @@(methodical/defmethod t2/table-name :model/Transform [_model] :transform)-(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped?]]+(doseq [trait [:metabase/model :hook/entity-id :hook/timestamped? :hook/dependencies]](derive :model/Transform trait))(defn- transform-readable?diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex a96803cf2cf..093888530da 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -8,6 +8,7 @@[java-time.api :as t][metabase.app-db.cluster-lock :as cluster-lock][metabase.database-routing.core :as database-routing]+ [metabase.dependencies.hooks :as dependencies.hooks][metabase.driver :as driver][metabase.events.core :as events][metabase.indexes.models.table-index :as table-index]@@ -731,9 +732,10 @@{:keys [publish-events?]:or {publish-events? true}} optsdb-id (transforms-base.i/target-db-id transform)- database (transforms-base.db/database db-id)]+ database (transforms-base.db/database db-id)+ table (sync-target! target database)];; Sync target table, set target_table_id on transform, and mark table as owned by this transform- (when-let [table (sync-target! target database)]+ (when table(transforms-base.db/update-transform! (:id transform) {:target_table_id (:id table)})(transforms-base.db/update-table! (:id table) {:transform_id (:id transform)}));; ANALYZE the target before the run is observable, so dependents don't plan on stale stats.@@ -745,6 +747,7 @@(log/warnf "refresh-table-stats! failed for %s.%s: %s" (:schema target) (:name target) (ex-message t))));; Publish event after sync so the table exists in AppDB.(when publish-events?+ (dependencies.hooks/after-transform-run (:id transform) (:id table))(events/publish-event! :event/transform-run-complete{:object {:db-id db-id:transform-id (:id transform)diff --git a/src/metabase/warehouse_schema/models/dimension.clj b/src/metabase/warehouse_schema/models/dimension.cljindex fc5117ae389..8b4e977f7b7 100644--- a/src/metabase/warehouse_schema/models/dimension.clj+++ b/src/metabase/warehouse_schema/models/dimension.clj@@ -20,7 +20,8 @@(doto :model/Dimension(derive :metabase/model)(derive :hook/entity-id)- (derive :hook/timestamped?))+ (derive :hook/timestamped?)+ (derive :hook/dependencies))(t2/deftransforms :model/Dimension{:type mi/transform-keyword})diff --git a/src/metabase/warehouse_schema/models/field_user_settings.clj b/src/metabase/warehouse_schema/models/field_user_settings.cljindex e3e5b140732..6cffef7dd62 100644--- a/src/metabase/warehouse_schema/models/field_user_settings.clj+++ b/src/metabase/warehouse_schema/models/field_user_settings.clj@@ -28,7 +28,8 @@(doto :model/FieldUserSettings(derive :metabase/model)- (derive :hook/timestamped?))+ (derive :hook/timestamped?)+ (derive :hook/dependencies))(defn- with-set-flags"Set the `_set` flag of every flagged column `effective` writes, unless `effective` sets the flag itself."diff --git a/src/metabase/warehouse_schema/models/table_user_settings.clj b/src/metabase/warehouse_schema/models/table_user_settings.cljindex 63bdcdd39f2..da5d56f8986 100644--- a/src/metabase/warehouse_schema/models/table_user_settings.clj+++ b/src/metabase/warehouse_schema/models/table_user_settings.clj@@ -34,7 +34,8 @@(doto :model/TableUserSettings(derive :metabase/model)(derive :hook/timestamped?)- (derive :hook/search-index))+ (derive :hook/search-index)+ (derive :hook/dependencies))(methodical/defmethod t2/primary-keys :model/TableUserSettings [_model] [:table_id])diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 47f58049e96..ff83a1fe0d0 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -69,6 +69,7 @@(doto :model/Database(derive :metabase/model)(derive :hook/timestamped?)+ (derive :hook/dependencies);; Deliberately **not** deriving from `:hook/entity-id` because we should not be randomizing the `entity_id`s on;; databases, tables or fields. Since the sync process can create them in multiple instances, randomizing them would;; cause duplication rather than good matching if the two instances are later linked by serdes.@@ -164,16 +165,6 @@[_db-id](mi/superuser?))-(defenterprise mark-transforms-stale-on-database-delete!- "Hook called from the `:model/Database` before-delete. Transforms whose `source_database_id`- is about to be SET NULL by the FK action survive the delete (so the analyst can read the- SQL/Python and rebuild against a new DB), but their dependency-analysis findings need to- be re-run so they surface on `/monitor/dependency-diagnostics/broken`. OSS is a no-op- (no dependencies module)."- metabase-enterprise.dependencies.events- [_db-id]- nil)-(defn- can-write?[db-id](or (some-> db-id db-id->router-db-id can-write?)@@ -433,11 +424,6 @@(t2/define-before-delete :model/Database[{id :id, driver :engine, :as database}]- ;; Mark transforms with this DB as source as stale so dependency-diagnostics re-analyzes- ;; them after the row's gone. Must run BEFORE the FK SET NULL fires (which happens when- ;; the database row is deleted below) so we can still resolve the affected transforms by- ;; source_database_id.- (mark-transforms-stale-on-database-delete! id)(unschedule-tasks! database)(secret/delete-orphaned-secrets! database)(delete-database-fields! id)diff --git a/test/metabase/dashboards/models/dashboard_card_test.clj b/test/metabase/dashboards/models/dashboard_card_test.cljindex 67348e63262..21267e24445 100644--- a/test/metabase/dashboards/models/dashboard_card_test.clj+++ b/test/metabase/dashboards/models/dashboard_card_test.clj@@ -250,8 +250,8 @@:size_x 3:size_y 4:series []}])- ;; this is usually 10, but it can be 11 sometimes in CI for some reason- (is (contains? #{10 11} (call-count)))))))))+ ;; this is usually 13, but it can be 14 sometimes in CI for some reason+ (is (contains? #{13 14} (call-count)))))))))(deftest ^:parallel normalize-parameter-mappings-test(testing "DashboardCard parameter mappings should get normalized when coming out of the DB"diff --git a/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yaml b/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yamlindex fb6010b92ba..68df1e121db 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/click_behavior_examples.yaml@@ -33,8 +33,7 @@ dashcards:- CATEGORY- nullseries: []- visualization_settings:- column_settings: null+ visualization_settings: {}- entity_id: iMNmMMABHvbbiVT3ufez6card_id: UEWVldEEZ-3sYUjjN54qzcreated_at: '2026-04-02T02:59:02.778668Z'@@ -45,8 +44,7 @@ dashcards:inline_parameters: []parameter_mappings: []series: []- visualization_settings:- column_settings: null+ visualization_settings: {}- entity_id: RasgF0LvnoJlu7ModLrqvcard_id: dYD1j3pQtJdctg8duByULcreated_at: '2026-04-02T02:59:02.778668Z'@@ -57,8 +55,7 @@ dashcards:inline_parameters: []parameter_mappings: []series: []- visualization_settings:- column_settings: null+ visualization_settings: {}- entity_id: NSyK1LJldomowKbt0HpuEcard_id: ciG63nSLHKRTImt5iWEtecreated_at: '2026-04-02T02:59:02.778668Z'@@ -69,7 +66,6 @@ dashcards:inline_parameters: []parameter_mappings: []series: []- visualization_settings:- column_settings: null+ visualization_settings: {}tabs: []width: fixeddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yaml b/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yamlindex e095dc1e53b..eefda2f100c 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/inline_parameters_dashboard.yaml@@ -31,7 +31,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nulltext: Products in {{category}}virtual_card:display: heading@@ -47,7 +46,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nulltext: |-Showing products in the **{{category}}** categorywith price above **{{min_price}}**.@@ -83,7 +81,6 @@ dashcards:- PRICE- nullseries: []- visualization_settings:- column_settings: null+ visualization_settings: {}tabs: []width: fixeddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yamlindex a016901964c..3f85c089696 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings.yaml@@ -56,7 +56,6 @@ dashcards:- card_id: dShCrdMnthlyCumul0004position: 2visualization_settings:- column_settings: nullgraph.show_values: truegraph.x_axis.title_text: Monthgraph.y_axis.title_text: Revenue@@ -76,8 +75,7 @@ dashcards:- - template-tag- min_totalseries: []- visualization_settings:- column_settings: null+ visualization_settings: {}- entity_id: f9pAJKJNxCtrS93uIN7F4card_id: dShCrdTtlRevScalr0006created_at: '2026-04-02T02:59:02.778668Z'@@ -89,7 +87,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nullscalar.field: Total Revenuetabs: []width: fixeddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yamlindex f9bf8edcc19..ec99e36c388 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_average_order.yaml@@ -31,8 +31,7 @@ dataset_query:- ORDERSlib/type: mbql.stage/mbqllib/type: mbql/query-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yamlindex 1b93ceb1677..2e111c34dbf 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_cumulative_orders.yaml@@ -76,8 +76,7 @@ result_metadata:lib/original-name: countlib/source: source/aggregationslib/source-column-alias: count-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yamlindex 8819b743f00..8bfedce93bf 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_orders.yaml@@ -76,8 +76,7 @@ result_metadata:lib/original-name: countlib/source: source/aggregationslib/source-column-alias: count-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yamlindex 33a1eacf59d..3a28abca692 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/monthly_revenue.yaml@@ -81,8 +81,7 @@ result_metadata:lib/original-name: sumlib/source: source/aggregationslib/source-column-alias: sum-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yamlindex 9003027709f..a06383061b3 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/native_orders_query.yaml@@ -19,8 +19,7 @@ dataset_query:type: numberlib/type: mbql.stage/nativelib/type: mbql/query-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yaml b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yamlindex 2c194750480..abbbf50c0a2 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/series_and_parameter_mappings/total_revenue.yaml@@ -38,8 +38,7 @@ result_metadata:lib/original-name: sumlib/source: source/aggregationslib/source-column-alias: sum-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yaml b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yamlindex c4d04aa2ed0..3a45faaea4d 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard.yaml@@ -36,8 +36,7 @@ dashcards:- CREATED_AT- nullseries: []- visualization_settings:- column_settings: null+ visualization_settings: {}- entity_id: uyqXk5DNhkAMTp4W1TD9-card_id: dShCrdProdsTables0007created_at: '2026-04-02T02:59:02.778668Z'@@ -51,8 +50,7 @@ dashcards:inline_parameters: []parameter_mappings: []series: []- visualization_settings:- column_settings: null+ visualization_settings: {}tabs:- created_at: '2026-04-02T02:59:02.778668Z'entity_id: XcmNByL-bsDQ15jLHBUwrdiff --git a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yaml b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yamlindex 12e5ffdb3cf..92de3037adb 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/tabbed_dashboard/products_table.yaml@@ -245,8 +245,7 @@ result_metadata:lib/source: source/table-defaultslib/source-column-alias: RATINGqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yaml b/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yamlindex 6f17f573b4b..ff7d0169779 100644--- a/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yaml+++ b/test_resources/serialization_baseline/collections/main/dashboards/virtual_cards_dashboard.yaml@@ -17,7 +17,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nulltext: Sales Overviewvirtual_card:display: heading@@ -31,7 +30,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nulltext: |-This dashboard provides an overview of sales metrics.**Key metrics** include revenue, order count, and average order value.@@ -48,7 +46,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nulllink:url: https://metabase.com/docsvirtual_card:@@ -63,7 +60,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nulllink:entity:id: Q_jD-f-9clKLFZ2TfUG2h@@ -80,7 +76,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nulliframe: <iframe src="https://example.com/embed" width="100%" height="400"></iframe>virtual_card:display: iframe@@ -94,7 +89,6 @@ dashcards:parameter_mappings: []series: []visualization_settings:- column_settings: nullvirtual_card:display: placeholdertabs: []diff --git a/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yaml b/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yamlindex 963cfefab65..e13c857eb09 100644--- a/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yaml+++ b/test_resources/serialization_baseline/collections/main/documents/product_analysis_report/category_breakdown.yaml@@ -25,8 +25,7 @@ dataset_query:- PRODUCTSlib/type: mbql.stage/mbqllib/type: mbql/query-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yaml b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yamlindex 8ecc04be211..ac36ab0ae9a 100644--- a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yaml+++ b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_dashboard.yaml@@ -33,7 +33,6 @@ dashcards:- CATEGORY- nullseries: []- visualization_settings:- column_settings: null+ visualization_settings: {}tabs: []width: fixeddiff --git a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yaml b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yamlindex 5f6e74255a8..232f317d9fb 100644--- a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yaml+++ b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/product_overview.yaml@@ -122,8 +122,7 @@ result_metadata:lib/original-name: avglib/source: source/aggregationslib/source-column-alias: avg-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yaml b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yamlindex bba364bf26d..87fde29ba14 100644--- a/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yaml+++ b/test_resources/serialization_baseline/collections/main/minimal/marketing_analytics/top_products_by_revenue.yaml@@ -28,8 +28,7 @@ dataset_query:type: textlib/type: mbql.stage/nativelib/type: mbql/query-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yaml b/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yamlindex 30f746bce93..6cd2a8a7f97 100644--- a/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/arithmetic_expressions.yaml@@ -204,8 +204,7 @@ result_metadata:lib/original-name: avglib/source: source/aggregationslib/source-column-alias: avg-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yaml b/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yamlindex e0f95b1f883..db5c8e644c2 100644--- a/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/basic_aggregations.yaml@@ -238,8 +238,7 @@ result_metadata:lib/original-name: total_revenuelib/source: source/aggregationslib/source-column-alias: total_revenue-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yaml b/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yamlindex 231fa0885e6..5938541eb3f 100644--- a/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/comparison_filters.yaml@@ -227,8 +227,7 @@ result_metadata:lib/source: source/table-defaultslib/source-column-alias: RATINGqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yaml b/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yamlindex 43a0be2f890..edfcd25de71 100644--- a/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/conditional_aggregations.yaml@@ -169,8 +169,7 @@ result_metadata:lib/original-name: sharelib/source: source/aggregationslib/source-column-alias: share-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yaml b/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yamlindex 22f54a0326b..1a540768cfe 100644--- a/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/conditional_and_type_conversion.yaml@@ -466,8 +466,7 @@ result_metadata:lib/source: source/expressionslib/source-column-alias: Rating as Floatqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yaml b/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yamlindex 6b3e0c36a68..7191894b9e6 100644--- a/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/cumulative_and_statistical_aggregations.yaml@@ -204,8 +204,7 @@ result_metadata:lib/original-name: percentilelib/source: source/aggregationslib/source-column-alias: percentile-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yaml b/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yamlindex c55eb7de448..d04e87aab07 100644--- a/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/fields_and_expressions.yaml@@ -162,8 +162,7 @@ result_metadata:lib/original-name: Profitlib/source: source/expressionslib/source-column-alias: Profit-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yaml b/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yamlindex 9c837611c06..b76249c8bc2 100644--- a/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/geographic_filter__inside_.yaml@@ -428,8 +428,7 @@ result_metadata:lib/source: source/table-defaultslib/source-column-alias: CREATED_ATqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yaml b/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yamlindex e6ccefde6aa..9bdd0cbbd46 100644--- a/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/joins___strategies_and_compound_conditions.yaml@@ -662,8 +662,7 @@ result_metadata:lib/source: source/joinslib/source-column-alias: STATEqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/math_functions.yaml b/test_resources/serialization_baseline/collections/main/queries/math_functions.yamlindex 52ad67c1a83..c549d185fc9 100644--- a/test_resources/serialization_baseline/collections/main/queries/math_functions.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/math_functions.yaml@@ -468,8 +468,7 @@ result_metadata:lib/source: source/expressionslib/source-column-alias: Price Logqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yaml b/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yamlindex c2e13b80849..9a278818933 100644--- a/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/metric__measure__and_segment_references.yaml@@ -54,8 +54,7 @@ result_metadata:lib/original-name: countlib/source: source/aggregationslib/source-column-alias: count-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yamlindex 7617397f76f..186a2991b4e 100644--- a/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___card_and_snippet_references.yaml@@ -29,8 +29,7 @@ dataset_query:type: snippetlib/type: mbql.stage/nativelib/type: mbql/query-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yamlindex 088d5c613fd..faac565760d 100644--- a/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___field_filter_and_temporal_unit.yaml@@ -56,8 +56,7 @@ dataset_query:widget-type: date/rangelib/type: mbql.stage/nativelib/type: mbql/query-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yaml b/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yamlindex 6d7eb059102..97a28b6bb62 100644--- a/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/native_query___variables.yaml@@ -43,8 +43,7 @@ dataset_query:type: textlib/type: mbql.stage/nativelib/type: mbql/query-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/nested_query.yaml b/test_resources/serialization_baseline/collections/main/queries/nested_query.yamlindex 9e910fd0549..b6f176a10ab 100644--- a/test_resources/serialization_baseline/collections/main/queries/nested_query.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/nested_query.yaml@@ -121,8 +121,7 @@ result_metadata:lib/source: source/previous-stagelib/source-column-alias: sumqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yaml b/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yamlindex 65cea0637a3..1b5628c9337 100644--- a/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/null__empty__and_string_filters.yaml@@ -416,8 +416,7 @@ result_metadata:lib/source: source/table-defaultslib/source-column-alias: CREATED_ATqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yaml b/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yamlindex c80398a533c..bd2280b382d 100644--- a/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/source_card_reference.yaml@@ -202,8 +202,7 @@ result_metadata:lib/source: source/previous-stagelib/source-column-alias: total_revenueqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/string_functions.yaml b/test_resources/serialization_baseline/collections/main/queries/string_functions.yamlindex ebdd187e5d9..a9fd17e8f35 100644--- a/test_resources/serialization_baseline/collections/main/queries/string_functions.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/string_functions.yaml@@ -547,8 +547,7 @@ result_metadata:lib/source: source/expressionslib/source-column-alias: Title From 5th Charqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yaml b/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yamlindex 1610bfb18fe..4ae50ccda2e 100644--- a/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/temporal_and_logical_filters.yaml@@ -299,8 +299,7 @@ result_metadata:lib/source: source/table-defaultslib/source-column-alias: QUANTITYqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yaml b/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yamlindex ddf85153438..13d67cc49af 100644--- a/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/temporal_expressions.yaml@@ -459,8 +459,7 @@ result_metadata:lib/source: source/expressionslib/source-column-alias: Today Dateqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yaml b/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yamlindex 418f90de7a3..9d2508cb92d 100644--- a/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/temporal_extraction.yaml@@ -702,8 +702,7 @@ result_metadata:lib/source: source/expressionslib/source-column-alias: Extracted Day of Week ISOqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/total_revenue.yaml b/test_resources/serialization_baseline/collections/main/queries/total_revenue.yamlindex d79141ee972..2c0b92553a3 100644--- a/test_resources/serialization_baseline/collections/main/queries/total_revenue.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/total_revenue.yaml@@ -38,8 +38,7 @@ result_metadata:lib/original-name: sumlib/source: source/aggregationslib/source-column-alias: sum-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/url_functions.yaml b/test_resources/serialization_baseline/collections/main/queries/url_functions.yamlindex 039b60a0b52..25593e35252 100644--- a/test_resources/serialization_baseline/collections/main/queries/url_functions.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/url_functions.yaml@@ -535,8 +535,7 @@ result_metadata:lib/source: source/expressionslib/source-column-alias: Source Pathqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yaml b/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yamlindex b8324b27065..5ae4a336044 100644--- a/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yaml+++ b/test_resources/serialization_baseline/collections/main/queries/window_function___offset.yaml@@ -105,8 +105,7 @@ result_metadata:lib/original-name: offsetlib/source: source/aggregationslib/source-column-alias: offset-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/source_model.yaml b/test_resources/serialization_baseline/collections/main/source_model.yamlindex d53a8979038..6c6440f1cdf 100644--- a/test_resources/serialization_baseline/collections/main/source_model.yaml+++ b/test_resources/serialization_baseline/collections/main/source_model.yaml@@ -319,8 +319,7 @@ result_metadata:lib/source: source/table-defaultslib/source-column-alias: QUANTITYqp/implicit-field?: true-visualization_settings:- column_settings: null+visualization_settings: {}card_schema: 24serdes/meta:- model: Carddiff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yamlindex 54a9ae16704..bd630194036 100644--- a/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yaml+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/customer_locations__map_.yaml@@ -442,7 +442,6 @@ result_metadata:lib/source-column-alias: CREATED_ATqp/implicit-field?: truevisualization_settings:- column_settings: nullmap.center_latitude: 39.5map.center_longitude: -98.35map.latitude_column: LATITUDEdiff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yamlindex f69cdb17d03..7458f9a7fec 100644--- a/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yaml+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/monthly_revenue_changes__waterfall_.yaml@@ -82,7 +82,6 @@ result_metadata:lib/source: source/aggregationslib/source-column-alias: sumvisualization_settings:- column_settings: nullwaterfall.decrease_color: '#EF8C8C'waterfall.increase_color: '#88BF4D'waterfall.show_total: truediff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yamlindex a20b62ae694..d4f193a3a24 100644--- a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yaml+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__pie_.yaml@@ -74,7 +74,6 @@ result_metadata:lib/source: source/aggregationslib/source-column-alias: countvisualization_settings:- column_settings: nullpie.colors:Doohickey: '#ED6E6E'Gadget: '#88BF4D'diff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yamlindex ba255c9cfe9..58c38495132 100644--- a/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yaml+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/orders_by_category__stacked_bar_.yaml@@ -103,7 +103,6 @@ result_metadata:lib/source: source/aggregationslib/source-column-alias: countvisualization_settings:- column_settings: nullgraph.dimensions:- CREATED_AT- CATEGORYdiff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yamlindex 91646a28645..c067e50e615 100644--- a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yaml+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_gauge.yaml@@ -39,7 +39,6 @@ result_metadata:lib/source: source/aggregationslib/source-column-alias: sumvisualization_settings:- column_settings: nullgauge.segments:- color: '#ED6E6E'label: Lowdiff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yamlindex 0a054033de4..d5c11b88491 100644--- a/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yaml+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/revenue_over_time__line_.yaml@@ -82,7 +82,6 @@ result_metadata:lib/source: source/aggregationslib/source-column-alias: sumvisualization_settings:- column_settings: nullgraph.goal_label: Monthly Targetgraph.goal_value: 50000graph.label_values_frequency: fitdiff --git a/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yaml b/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yamlindex 6bf2a12efac..b61cf639055 100644--- a/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yaml+++ b/test_resources/serialization_baseline/collections/main/visualization_settings/sales_funnel.yaml@@ -74,7 +74,6 @@ result_metadata:lib/source: source/aggregationslib/source-column-alias: countvisualization_settings:- column_settings: nullfunnel.dimension: CATEGORYfunnel.metric: countfunnel.type: funnel