The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Ghy-4069 minimal migration approach (#77297)
Ben Grabowf0ef171d2587
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 24cf1b6b213..51db75d23fb 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1947,19 +1947,11 @@{:team "UX West":api #{metabase.sample-data.coremetabase.sample-data.init}- :uses #{config- models- permissions- plugins+ :uses #{pluginssettingssyncutil}- :model-imports #{:model/Card- :model/Dashboard- :model/DashboardCard- :model/DataPermissions- :model/Database- :model/Field+ :model-imports #{:model/Database:model/Table}}searchdiff --git a/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml b/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yamlindex 603f5c154a1..3c975258f4b 100644--- a/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml+++ b/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml@@ -1,12 +1,10 @@-## On downgrade from a SQLite-sample-database version to an H2-sample-database version, remove the-## SQLite sample database and the content left dangling by it. Does nothing on rollforward; the-## H2 -> SQLite upgrade replacement is handled at startup in metabase.sample-data.impl.+## No-op; superseded by the in-place sample-database engine swap at startup (metabase.sample-data.impl).databaseChangeLog:- changeSet:id: v63.2026-06-08T00:00:00author: bgrabow- comment: Remove the SQLite sample database (and content it leaves empty) on downgrade to an H2 version+ comment: No-op; sample database engine swap is handled at startupvalidCheckSum: ANYchanges:- customChange:diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 2fdb9042497..545524031e0 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -1330,120 +1330,9 @@(custom-migrations.util/with-temp-schedule! [scheduler](qs/delete-job scheduler (jobs/key "metabase.task.send-pulses.init-send-pulse-triggers.job"))))-(defn- card-ids-depending-on-database- "Set of report_card ids that depend on `database-id` directly (their own query targets it) or transitively- (a card built on such a card via source_card_id, to any depth)."- [database-id]- (loop [frontier (into #{} (map :id) (t2/query {:select [:id] :from [:report_card]- :where [:= :database_id database-id]}))- acc #{}]- (if (empty? frontier)- acc- (let [acc' (into acc frontier)- next* (into #{} (map :id) (t2/query {:select [:id] :from [:report_card]- :where [:and [:in :source_card_id frontier]- [:not [:in :id acc']]]}))]- (recur next* acc')))))--(defn- collection-has-survivors?- "True if `coll-id` still holds anything after sample content is gone: any content row filed into it, or any- surviving child collection (whose path is exactly `child-location`). Checked bottom-up so a child that itself- survived keeps its parent alive."- [coll-id child-location]- (boolean- (or (seq (t2/query {:select [1] :from [:collection] :where [:= :location child-location] :limit 1}))- (some (fn [table]- (seq (t2/query {:select [1] :from [table] :where [:= :collection_id coll-id] :limit 1})))- [:report_card :report_dashboard :native_query_snippet :pulse :timeline :document- :metabase_table :transform]))))--(defn- remove-sqlite-sample-database-on-downgrade!- "Delete the SQLite sample database and everything that depends on it - its tables/fields, every Card whose- query targets it (directly or transitively through source_card_id), those cards' dashcards, and Dashboards- left empty as a result - even when that content was user-created, because it cannot work without the database.-- The Example collections are then pruned, bottom-up: a sample collection is deleted only if nothing is left in- it once the sample content is gone (no content rows, no surviving child collection); a collection a user kept- their own content in is left intact. Permission records for the deleted collections go with them.-- Children are deleted explicitly, bottom-up, rather than relying on ON DELETE CASCADE: MySQL 9.7 resolves- multi-level cascade fan-outs incompletely, leaving orphaned rows that later break ALTER TABLE statements which- re-validate foreign keys."- []- (when-let [sample-db-id (:id (t2/query-one {:select [:id]- :from [:metabase_database]- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))]- (let [card-ids (card-ids-depending-on-database sample-db-id)- field-ids-q {:select [:id]- :from [:metabase_field]- :where [:in :table_id {:select [:id]- :from [:metabase_table]- :where [:= :db_id sample-db-id]}]}- affected-dashboard-ids (if (empty? card-ids)- #{}- (->> (t2/query {:select-distinct [:dashboard_id]- :from [:report_dashboardcard]- :where [:in :card_id card-ids]})- (into #{} (map :dashboard_id))))]- (when (seq card-ids)- (t2/query {:delete-from :dashboardcard_series- :where [:or- [:in :card_id card-ids]- [:in :dashboardcard_id {:select [:id]- :from [:report_dashboardcard]- :where [:in :card_id card-ids]}]]})- (t2/query {:delete-from :report_dashboardcard :where [:in :card_id card-ids]})- (t2/query {:delete-from :parameter_card- :where [:or- [:in :card_id card-ids]- [:and [:= :parameterized_object_type "card"]- [:in :parameterized_object_id card-ids]]]})- (t2/query {:delete-from :report_card :where [:in :id card-ids]}))- (t2/query {:delete-from :dimension :where [:in :field_id field-ids-q]})- (t2/query {:delete-from :metabase_field- :where [:in :table_id {:select [:id]- :from [:metabase_table]- :where [:= :db_id sample-db-id]}]})- (t2/query {:delete-from :metabase_table :where [:= :db_id sample-db-id]})- (t2/query {:delete-from :metabase_database :where [:= :id sample-db-id]})- (when (seq affected-dashboard-ids)- ;; A dashboard that depended on the sample DB is removed once no card-backed dashcard survives - even if- ;; text/heading dashcards remain (the example dashboard is built entirely from sample cards plus headings).- ;; A dashboard that still has a real card of its own (e.g. a user mixed in another card) is left alone.- (let [non-empty (->> (t2/query {:select-distinct [:dashboard_id]- :from [:report_dashboardcard]- :where [:and [:in :dashboard_id affected-dashboard-ids]- [:not= :card_id nil]]})- (into #{} (map :dashboard_id)))- empty-ids (remove non-empty affected-dashboard-ids)]- (when (seq empty-ids)- (t2/query {:delete-from :report_dashboardcard :where [:in :dashboard_id empty-ids]})- (t2/query {:delete-from :dashboard_tab :where [:in :dashboard_id empty-ids]})- (t2/query {:delete-from :parameter_card- :where [:and [:= :parameterized_object_type "dashboard"]- [:in :parameterized_object_id empty-ids]]})- (t2/query {:delete-from :report_dashboard :where [:in :id empty-ids]}))))- ;; Prune the Example collections deepest-first, so an emptied child is gone before its parent is judged.- (doseq [{:keys [id location]} (->> (t2/query {:select [:id :location] :from [:collection] :where [:= :is_sample true]})- (sort-by (comp count :location) >))]- (when-not (collection-has-survivors? id (str location id "/"))- (t2/query {:delete-from :permissions- :where [:or- [:= :collection_id id]- [:in :object [(format "/collection/%d/" id)- (format "/collection/%d/read/" id)]]]})- (t2/query {:delete-from :collection :where [:= :id id]}))))))--;; The bundled sample database moved from H2 to SQLite. If an instance running a SQLite-sample-;; version is downgraded to an H2-sample version, the older code cannot use the SQLite sample-;; database (it has the wrong engine/details) and would leave it broken. On downgrade we therefore-;; remove the SQLite sample database and the content left dangling by it, and we do NOT restore an-;; H2 sample database. This mirrors the upgrade behavior in-;; metabase.sample-data.impl/replace-sample-database!. There is no forward migration: the upgrade-;; (H2 -> SQLite) replacement is handled at startup, not here.-(define-reversible-migration MigrateAwayFromSqliteSampleDatabaseOnDowngrade- (log/info "No forward migration for MigrateAwayFromSqliteSampleDatabaseOnDowngrade")- (remove-sqlite-sample-database-on-downgrade!))+;; No-op. The SQLite <-> H2 sample database engine swap is handled in place at startup+;; (metabase.sample-data.impl). Kept because a shipped v63 build ran a delete-based rollback here.+(define-reversible-migration MigrateAwayFromSqliteSampleDatabaseOnDowngrade nil nil);; when card display is area or bar,;; 1. set the display key to :stackable.stack_display value OR leave it the samediff --git a/src/metabase/sample_data/example_content.clj b/src/metabase/sample_data/example_content.cljdeleted file mode 100644index 06e1fc65cfa..00000000000--- a/src/metabase/sample_data/example_content.clj+++ /dev/null@@ -1,285 +0,0 @@-(ns metabase.sample-data.example-content- "Recreate the bundled Example collection (the content shipped in `sample-content.edn`) on top of an- already-synced Sample Database.-- On a fresh install `metabase.app-db.custom-migrations/CreateSampleContentV2` inserts the EDN rows- verbatim into empty tables, so auto-increment reproduces the EDN ids exactly and every embedded- reference still lines up. On upgrade the tables are not empty, so we insert the same rows but remap- each id: simple FK columns by lookup, and the Database/Table/Field/Card ids embedded in card and- dashcard query blobs via the serdes export walkers bound to numeric id maps."- (:require- [clojure.edn :as edn]- [clojure.java.io :as io]- [clojure.string :as str]- [clojure.walk :as walk]- [medley.core :as m]- [metabase.models.interface :as mi]- [metabase.models.serialization :as serdes]- [metabase.settings.core :as setting]- [metabase.util.date-2 :as u.date]- [metabase.util.json :as json]- [metabase.util.log :as log]- [toucan2.core :as t2]))--(set! *warn-on-reflection* true)--(def ^:private sample-content-resource "sample-content.edn")--(defn- load-sample-content []- (with-open [r (io/reader (io/resource sample-content-resource))]- (edn/read {:readers {'t u.date/parse}} (java.io.PushbackReader. r))))--(defn- now-for-temporals [row]- (update-vals row (fn [v] (if (instance? java.time.temporal.Temporal v) :%now v))))--(defn- prep-row- "Strip the EDN id (let the app DB assign one) and replace temporal literals with `:%now`."- [row]- (-> row now-for-temporals (dissoc :id)))--(defn- insert-rows!- "Insert `rows` (each carrying its original EDN `:id` and `:entity_id`) into `table` after prepping,- and return a map of `edn-id -> new id` so other rows' numeric FK references can be remapped. The new- id is recovered by matching the unique `:entity_id` back to the inserted row."- [table rows]- (when (seq rows)- (t2/query {:insert-into table :values (mapv prep-row rows)}))- (let [eid->new (into {}- (map (juxt :entity_id :id))- (t2/query {:select [:id :entity_id] :from table- :where [:in :entity_id (mapv :entity_id rows)]}))]- (into {} (map (fn [r] [(:id r) (eid->new (:entity_id r))])) rows)))--;;; --------------------------------------------- id remapping ------------------------------------------------(defn- model->map-key [model]- (case (name model)- "Card" :cards- "Dashboard" :dashboards- "Collection" :collections- nil))--(defn- remap-result-metadata [maps cols]- (mapv (fn [m]- (-> m- (m/update-existing :table_id #(get (:tables maps) % %))- (m/update-existing :id #(if (number? %) (get (:fields maps) % %) %))- (m/update-existing :field_ref serdes/export-mbql)- (m/update-existing :fk_target_field_id #(if (number? %) (get (:fields maps) % %) %))))- cols))--(defn- remap-source-card-refs- "`export-mbql` only remaps a numeric `:source-table` (a table id); a `\"card__N\"` `:source-table` - a card- that builds on another card - is left carrying the EDN card id. Rewrite each through the card id map so it- points at the recreated card instead of whatever row happens to occupy the EDN id on an upgraded app db."- [card-map query]- (walk/postwalk- (fn [x]- (if-let [[_ n] (and (map? x)- (string? (:source-table x))- (re-matches #"card__(\d+)" (:source-table x)))]- (let [old-id (parse-long n)]- (assoc x :source-table (str "card__" (get card-map old-id old-id))))- x))- query))--(defn- remap-blob- "Remap the entity ids embedded in a serialized JSON `blob` string of the given `kind`, returning a- JSON string. Reuses the serdes export walkers with the FK resolvers bound to return new numeric ids- (rather than portable references)."- [maps kind blob]- (when (some? blob)- (binding [serdes/*export-database-fk* (fn [_id] (:db maps))- serdes/*export-field-fk* (fn [id] (get (:fields maps) id id))- serdes/*export-table-fk* (fn [id] (get (:tables maps) id id))- serdes/*export-fk* (fn [id model]- (let [k (model->map-key model)]- (if k (get (get maps k) id id) id)))]- (json/encode- (case kind- :mbql (->> (json/decode+kw blob) serdes/export-mbql (remap-source-card-refs (:cards maps)))- ;; Remap each parameter's refs in place, preserving the designed order. (export-parameters would reorder- ;; them by :id - a serialization-stability device we must not inherit, or the filter order is scrambled.)- :parameters (mapv serdes/export-mbql (json/decode+kw blob))- :param-maps (serdes/export-parameter-mappings (json/decode+kw blob))- ;; column_settings keys must stay strings, so decode without keywordizing (as the model does).- :viz (serdes/export-visualization-settings (mi/normalize-visualization-settings (json/decode blob)))- :result-md (remap-result-metadata maps (json/decode+kw blob)))))))--;;; --------------------------------------------- insertion ------------------------------------------------(defn- field-id-maps- "Build `{:db new-db-id :tables {edn-table-id new-table-id} :fields {edn-field-id new-field-id}}` by- matching the EDN table/field names against the freshly-synced Sample Database."- [content new-db-id]- (let [edn-tables (:metabase_table content)- edn-fields (:metabase_field content)- tname->new (t2/select-fn->pk :name :model/Table :db_id new-db-id)- table-id-map (into {} (keep (fn [{:keys [id name]}]- (when-let [new-id (tname->new name)]- [id new-id])))- edn-tables)- ;; [new-table-id field-name] -> new-field-id- new-fields (t2/select :model/Field :table_id [:in (vals table-id-map)])- nf-index (into {} (map (fn [f] [[(:table_id f) (:name f)] (:id f)])) new-fields)- field-id-map (into {} (keep (fn [{:keys [id table_id name]}]- (when-let [new-tid (table-id-map table_id)]- (when-let [new-fid (nf-index [new-tid name])]- [id new-fid])))- edn-fields))]- {:db new-db-id :tables table-id-map :fields field-id-map}))--(defn- remap-collection-location- "Remap the parent ids embedded in a collection `location` path (e.g. \"/2/3/\") through `old->new`, one path- segment at a time. Remapping per segment (rather than successive string replacement over the id map) avoids a- newly-assigned id being re-substituted as if it were a still-pending old id."- [old->new location]- (if (= "/" location)- location- (str "/"- (->> (str/split location #"/")- (remove str/blank?)- (map (fn [segment]- (if-let [old-id (parse-long segment)]- (str (get old->new old-id old-id))- segment)))- (str/join "/"))- "/")))--(defn- insert-collections! [content]- ;; Reuse the existing Example collections (matched by entity id) so user content filed into them survives an- ;; engine swap; only create one when it's missing. Roots before children so a new child's location path can be- ;; remapped against ids already assigned.- (loop [pending (sort-by (comp count :location) (:collection content))- coll-map {}]- (if-let [row (first pending)]- (let [existing (:id (t2/query-one {:select [:id] :from :collection :where [:= :entity_id (:entity_id row)]}))- new-id (or existing- (let [prepared (-> row prep-row (update :location #(remap-collection-location coll-map %)))]- (t2/query {:insert-into :collection :values [prepared]})- (:id (t2/query-one {:select [:id] :from :collection :where [:= :entity_id (:entity_id row)]}))))]- (recur (rest pending) (assoc coll-map (:id row) new-id)))- coll-map)))--(defn- delete-prior-sample-content!- "Drop the previously-bundled sample cards and dashboards, matched by their stable EDN entity ids, so reinserting- them can't collide and stale ones don't linger. User content never carries these entity ids, so it is untouched;- the Example collections are reused, not deleted. Dashboards are removed before cards because deleting a dashboard- cascades its dashcards."- [content]- (doseq [table [:report_dashboard :report_card]]- (let [eids (into [] (keep :entity_id) (get content table))]- (when (seq eids)- (t2/query {:delete-from table :where [:in :entity_id eids]})))))--(defn- insert-cards! [content maps]- ;; Phase 1: insert with simple FKs remapped and blobs left as-is (no FK enforcement on JSON), so all- ;; card ids exist before we rewrite inter-card references in phase 2.- (let [coll-map (:collections maps)- rows (for [card (:report_card content)]- (-> card- (assoc :database_id (:db maps))- (assoc :collection_id (some-> (:collection_id card) coll-map))- (assoc :table_id (some-> (:table_id card) (->> (get (:tables maps)))))- (assoc :source_card_id nil)))- card-map (insert-rows! :report_card (vec rows))- maps (assoc maps :cards card-map)]- ;; Phase 2: rewrite query blobs + inter-card refs now that every card id exists.- (doseq [card (:report_card content)]- (t2/query {:update :report_card- :set {:dataset_query (remap-blob maps :mbql (:dataset_query card))- :result_metadata (remap-blob maps :result-md (:result_metadata card))- :parameters (remap-blob maps :parameters (:parameters card))- :parameter_mappings (remap-blob maps :param-maps (:parameter_mappings card))- :visualization_settings (remap-blob maps :viz (:visualization_settings card))- :source_card_id (some-> (:source_card_id card) card-map)}- :where [:= :id (card-map (:id card))]}))- maps))--(defn- insert-dashboards! [content maps]- (let [coll-map (:collections maps)- rows (mapv (fn [d] (assoc d :collection_id (some-> (:collection_id d) coll-map)))- (:report_dashboard content))- dash-map (insert-rows! :report_dashboard rows)- maps (assoc maps :dashboards dash-map)]- (doseq [d (:report_dashboard content)]- (t2/query {:update :report_dashboard- :set {:parameters (remap-blob maps :parameters (:parameters d))}- :where [:= :id (dash-map (:id d))]}))- maps))--(defn- insert-dashboard-tabs! [content maps]- (let [dash-map (:dashboards maps)- rows (mapv (fn [t] (assoc t :dashboard_id (dash-map (:dashboard_id t))))- (:dashboard_tab content))]- (assoc maps :tabs (insert-rows! :dashboard_tab rows))))--(defn- insert-dashcards! [content maps]- (let [{:keys [dashboards cards tabs]} maps- rows (mapv (fn [dc]- (-> (prep-row dc)- (assoc :dashboard_id (dashboards (:dashboard_id dc)))- (assoc :card_id (some-> (:card_id dc) cards))- (assoc :dashboard_tab_id (some-> (:dashboard_tab_id dc) tabs))- (assoc :parameter_mappings (remap-blob maps :param-maps (:parameter_mappings dc)))- (assoc :visualization_settings (remap-blob maps :viz (:visualization_settings dc)))))- (:report_dashboardcard content))]- (when (seq rows)- (t2/query {:insert-into :report_dashboardcard :values rows}))- maps))--(defn- insert-dimensions! [content maps]- (let [fields (:fields maps)- rows (mapv (fn [dim]- (-> (prep-row dim)- (assoc :field_id (fields (:field_id dim)))- (m/update-existing :human_readable_field_id #(some->> % (get fields)))))- (:dimension content))]- (when (seq rows)- (t2/query {:insert-into :dimension :values rows}))))--(defn- grant-example-collection-perms! [content maps]- (when-let [example-collection-id (some->> (:collection content)- (filter :is_sample)- first :id- (get (:collections maps)))]- (when-let [group-id (:id (t2/query-one {:select [:id] :from :permissions_group :where [:= :name "All Users"]}))]- ;; A reused Example collection already carries this grant; only insert when it's missing.- (when-not (seq (t2/query {:select [1] :from :permissions- :where [:and [:= :collection_id example-collection-id]- [:= :group_id group-id]- [:= :perm_type "perms/collection-access"]]- :limit 1}))- (t2/query {:insert-into :permissions- :values [{:object (format "/collection/%s/" example-collection-id)- :group_id group-id- :perm_type "perms/collection-access"- :perm_value "read-and-write"- :collection_id example-collection-id}]})))))--(defn- set-example-dashboard-id! [content maps]- (when-let [dash (first (:report_dashboard content))]- (when-let [new-id (get (:dashboards maps) (:id dash))]- (setting/set-value-of-type! :integer :example-dashboard-id new-id))))--(defn recreate-example-content!- "Recreate the bundled Example collection on the freshly-synced Sample Database `new-db-id`. Mirrors- the fresh-install seeding, remapping every id reference to point at the new database's tables and- fields. Best-effort: logs and swallows errors so a content-seeding failure can't break startup."- [new-db-id]- (try- (t2/with-transaction [_conn]- (let [content (load-sample-content)- _ (delete-prior-sample-content! content)- maps (field-id-maps content new-db-id)- maps (assoc maps :collections (insert-collections! content))- maps (insert-cards! content maps)- maps (insert-dashboards! content maps)- maps (insert-dashboard-tabs! content maps)- maps (insert-dashcards! content maps)]- (insert-dimensions! content maps)- (grant-example-collection-perms! content maps)- (set-example-dashboard-id! content maps)))- (log/info "Recreated Sample Database example content")- (catch Throwable e- (log/error e "Failed to recreate Sample Database example content"))))diff --git a/src/metabase/sample_data/impl.clj b/src/metabase/sample_data/impl.cljindex 678b61945be..1a3f229fca6 100644--- a/src/metabase/sample_data/impl.clj+++ b/src/metabase/sample_data/impl.clj@@ -3,10 +3,7 @@(:require[clojure.java.io :as io][clojure.string :as str]- [metabase.config.core :as config]- [metabase.permissions.core :as perms][metabase.plugins.core :as plugins]- [metabase.sample-data.example-content :as example-content][metabase.sync.core :as sync][metabase.util.files :as u.files][metabase.util.i18n :refer [trs]]@@ -112,118 +109,34 @@(catch Throwable e(log/error e "Failed to load sample database"))))-(defn- sample-database-dashboard-ids- "IDs of Dashboards holding at least one card backed by `database-id`. Captured before the sample- database is deleted so the dashboards it empties out can be pruned afterward."- [database-id]- (t2/select-fn-set :dashboard_id [:model/DashboardCard :dashboard_id]- :card_id [:in {:select [:id]- :from [(t2/table-name :model/Card)]- :where [:= :database_id database-id]}]))--(defn- delete-emptied-dashboards!- "Delete whichever of `dashboard-ids` no longer have any dashcards. A sample dashboard whose cards- were all removed with the sample database is deleted; a dashboard that still has cards (e.g. mixes- in cards from another database) is left alone."- [dashboard-ids]- (when (seq dashboard-ids)- (let [non-empty (t2/select-fn-set :dashboard_id [:model/DashboardCard :dashboard_id] :dashboard_id [:in dashboard-ids])- empty-ids (remove (or non-empty #{}) dashboard-ids)]- (when (seq empty-ids)- (t2/delete! :model/Dashboard :id [:in empty-ids])))))--(defn- capture-permissions-snapshot- "Snapshot a database's data-permission rows plus the names of the tables referenced by any table-level- rows, so the same access can be re-applied to a replacement database whose tables have new IDs. Must run- before the old database (and its cascading permission rows) is deleted."- [db-id]- (let [permissions (t2/select :model/DataPermissions :db_id db-id)- table-ids (into #{} (keep :table_id) permissions)]- {:permissions permissions- :table-id->name (if (seq table-ids)- (t2/select-pk->fn :name :model/Table :id [:in table-ids])- {})}))--(defn- apply-permissions-snapshot!- "Re-apply a permission snapshot (see [[capture-permissions-snapshot]]) to the replacement database- `new-db-id`, replacing the defaults granted when it was created. Table-level rows are remapped onto the- new database's tables by name; since the new tables exist only after the first sync, this must run- post-sync. A table-level row whose table has no name match in the new database is dropped."- [new-db-id {:keys [permissions table-id->name]}]- (let [name->new-table-id (t2/select-fn->pk :name :model/Table :db_id new-db-id)- granular? (fn [rows] (some :table_id rows))- permission-groups (group-by (juxt :group_id :perm_type) permissions)- db-perms (for [[[group-id perm-type] rows] permission-groups- :when (not (granular? rows))]- [group-id perm-type (:perm_value (first rows))])- table-perms (for [[[group-id perm-type] rows] permission-groups- :when (granular? rows)- :let [remapped (into {}- (keep (fn [{:keys [table_id perm_value]}]- (when-let [new-table-id (-> table_id table-id->name name->new-table-id)]- [new-table-id perm_value]))- rows))]- :when (seq remapped)]- [group-id perm-type remapped])]- (doseq [[group-id perm-type perm-value] db-perms]- (perms/set-database-permission! group-id new-db-id perm-type perm-value))- (doseq [[group-id perm-type one-table-perms] table-perms]- (perms/set-table-permissions! group-id perm-type one-table-perms))))--(defn- replace-sample-database!- "The bundled sample database's engine changed (e.g. H2 -> SQLite on upgrade). The old sample- Database's tables, fields, and connection details are incompatible with the new engine, so rather- than remapping content we drop the old sample Database wholesale and extract + sync the new one.- Deleting the Database cascades to its Cards and those cards' dashcards; Dashboards left empty as a- result are deleted too.-- The cleanup runs unconditionally (an incompatible sample database should never linger), but the new- Sample Database and its Example collection are only recreated when sample content is enabled - and- the Example collection is skipped under test endpoints, matching fresh-install seeding.-- The Example collection itself is left in place: [[example-content/recreate-example-content!]] reuses it- (matching by entity id) and replaces only the bundled cards/dashboards, so any content a user filed into- the Example collection survives the engine swap."+(defn- table-schema-for-engine+ "The schema value the sync process assigns to the sample database's tables for a given engine: H2 puts+ everything in the PUBLIC schema, SQLite reports no schema."+ [engine]+ (case engine+ :h2 "PUBLIC"+ :sqlite nil))++(defn- migrate-sample-database-engine-in-place!+ "The only app-db differences between the H2 and+ SQLite sample databases are the Database record's engine/details and the tables' schema (H2 = \"PUBLIC\",+ SQLite = nil). So instead of deleting and rebuilding the database, edit those two things in place and+ re-sync to reconcile the remaining sync-derived field metadata. Every Database/Table/Field id is kept,+ so sample content, user-created content, permissions, and the Example collection all survive with no+ remapping, deletion, or re-seeding."[engine old-sample-db]- (log/infof "Bundled sample database engine changed from %s to %s; replacing the sample database"- (:engine old-sample-db) engine)- ;; Extract the bundled DB file (a multi-MB copy out of the JAR) before opening the transaction, so the file IO- ;; doesn't hold the app-DB transaction open.- (let [new-db-details (when (config/load-sample-content?)- (try-to-extract-sample-database! engine))- {:keys [new-db permissions-snapshot]}- (t2/with-transaction [_conn]- (let [dashboard-ids (sample-database-dashboard-ids (:id old-sample-db))- snapshot (when new-db-details- (capture-permissions-snapshot (:id old-sample-db)))]- (t2/delete! :model/Database (:id old-sample-db))- (delete-emptied-dashboards! dashboard-ids)- {:permissions-snapshot snapshot- :new-db (when new-db-details- (first (t2/insert-returning-instances! :model/Database- :name sample-database-name- :details new-db-details- :engine engine- :is_sample true)))}))]- (when new-db- (let [synced? (try- (sync/sync-database! new-db)- true- (catch Throwable e- (log/error e "Failed to sync the replacement sample database")- false))]- (when synced?- ;; The new database's tables exist now, so the old sample DB's access can be restored (replacing- ;; the defaults granted at creation) before the bundled example content is recreated.- (when permissions-snapshot- (apply-permissions-snapshot! (:id new-db) permissions-snapshot))- (when-not (config/config-bool :mb-enable-test-endpoints)- (example-content/recreate-example-content! (:id new-db))))))))+ (log/infof "Migrating sample database engine from %s to %s in place" (:engine old-sample-db) engine)+ (let [details (try-to-extract-sample-database! engine)]+ (t2/with-transaction [_conn]+ (t2/update! :model/Database (:id old-sample-db) {:engine engine, :details details})+ (t2/update! :model/Table :db_id (:id old-sample-db) {:schema (table-schema-for-engine engine)}))+ (sync/sync-database! (t2/select-one :model/Database :id (:id old-sample-db)))))(defn update-sample-database-if-needed!"Reconcile the existing sample database with the bundled one. When the bundled engine changed- (H2 <-> SQLite) the old sample database is replaced (see [[replace-sample-database!]]); otherwise- we just refresh its connection details in case the JAR has moved."+ (H2 <-> SQLite) the existing sample database is migrated in place to the new engine (see+ [[migrate-sample-database-engine-in-place!]]); otherwise we just refresh its connection details in case+ the JAR has moved."([](update-sample-database-if-needed! (t2/select-one :model/Database :is_sample true)))@@ -231,7 +144,7 @@(when sample-db(let [engine (sample-database-engine)](if (not= (:engine sample-db) engine)- (replace-sample-database! engine sample-db)+ (migrate-sample-database-engine-in-place! engine sample-db)(let [intended (try-to-extract-sample-database! engine)](when (not= (:details sample-db) intended)(t2/update! :model/Database (:id sample-db) {:details intended}))))))))diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 1bf7dabd0c4..f587b8a790c 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -477,58 +477,53 @@(t2/define-before-update :model/Database[database]+ ;; Note: the "sample database may not be edited" policy is enforced at the API layer+ ;; ([[metabase.warehouses-rest.api]] PUT /:id), so internally-derived updates - e.g. the sample+ ;; database engine migration in [[metabase.sample-data.impl]] - can change the engine here.(let [changes (t2/changes database){new-engine :enginenew-settings :settings} changes- {is-sample? :is_sample- existing-settings :settings+ {existing-settings :settingsexisting-engine :engine} (t2/original database)new-engine (some-> new-engine keyword)]- (if (and is-sample?- new-engine- (not= new-engine existing-engine))- (throw (ex-info (trs "The engine on a sample database cannot be changed.")- {:status-code 400- :existing-engine existing-engine- :new-engine new-engine}))- (u/prog1 (cond-> database- ;; If the engine doesn't support nested field columns, `json_unfolding` must be nil- (and (some? (:details changes))- (not (driver.u/supports? (or new-engine existing-engine) :nested-field-columns database)))- (update :details dissoc :json_unfolding)-- (or- ;if there is any changes in user control setting- (some? (get-in changes [:details :let-user-control-scheduling]))- ;; if the let user control scheduling is already on, we should always try to re-infer it- (get-in database [:details :let-user-control-scheduling])- ;; if there is a changes in schedules, make sure it respects the settings- (some some? [(:cache_field_values_schedule changes) (:metadata_sync_schedule changes)]))- infer-db-schedules-- (or (some? (:details changes))- (some? (:write_data_details changes))- (some? (:admin_details changes)))- secret/handle-incoming-client-secrets!-- (:uploads_enabled changes)- maybe-disable-uploads-for-all-dbs!)- ;; This maintains a constraint that if a driver doesn't support actions, it can never be enabled- ;; If we drop support for actions for a driver, we'd need to add a migration to disable actions for all databases- (when (and (:database-enable-actions (or new-settings existing-settings))- (not (driver.u/supports? (or new-engine existing-engine) :actions database)))- (throw (ex-info (trs "The database does not support actions.")- {:status-code 400- :existing-engine existing-engine- :new-engine new-engine})))- ;; This maintains a constraint that if a driver doesn't support data editing, it can never be enabled- ;; If we drop support for a driver, we'd need to add a migration to disable it for all databases- (when (and (:database-enable-table-editing (or new-settings existing-settings))- (not (driver.u/supports? (or new-engine existing-engine) :actions/data-editing database)))- (throw (ex-info (trs "The database does not support table editing.")- {:status-code 400- :existing-engine existing-engine- :new-engine new-engine})))))))+ (u/prog1 (cond-> database+ ;; If the engine doesn't support nested field columns, `json_unfolding` must be nil+ (and (some? (:details changes))+ (not (driver.u/supports? (or new-engine existing-engine) :nested-field-columns database)))+ (update :details dissoc :json_unfolding)++ (or+ ;if there is any changes in user control setting+ (some? (get-in changes [:details :let-user-control-scheduling]))+ ;; if the let user control scheduling is already on, we should always try to re-infer it+ (get-in database [:details :let-user-control-scheduling])+ ;; if there is a changes in schedules, make sure it respects the settings+ (some some? [(:cache_field_values_schedule changes) (:metadata_sync_schedule changes)]))+ infer-db-schedules++ (or (some? (:details changes))+ (some? (:write_data_details changes))+ (some? (:admin_details changes)))+ secret/handle-incoming-client-secrets!++ (:uploads_enabled changes)+ maybe-disable-uploads-for-all-dbs!)+ ;; This maintains a constraint that if a driver doesn't support actions, it can never be enabled+ ;; If we drop support for actions for a driver, we'd need to add a migration to disable actions for all databases+ (when (and (:database-enable-actions (or new-settings existing-settings))+ (not (driver.u/supports? (or new-engine existing-engine) :actions database)))+ (throw (ex-info (trs "The database does not support actions.")+ {:status-code 400+ :existing-engine existing-engine+ :new-engine new-engine})))+ ;; This maintains a constraint that if a driver doesn't support data editing, it can never be enabled+ ;; If we drop support for a driver, we'd need to add a migration to disable it for all databases+ (when (and (:database-enable-table-editing (or new-settings existing-settings))+ (not (driver.u/supports? (or new-engine existing-engine) :actions/data-editing database)))+ (throw (ex-info (trs "The database does not support table editing.")+ {:status-code 400+ :existing-engine existing-engine+ :new-engine new-engine}))))))(t2/define-after-update :model/Database[database]diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex 33d7130f95f..56062d4608a 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -2938,200 +2938,3 @@(is (= "metabase-transform" (:data_source provisional)))(is (= "computed" (:data_authority provisional)))(is (= "New Target Table" (:display_name provisional)))))))))--(deftest ^:mb/old-migrations-test migrate-away-from-sqlite-sample-database-on-downgrade-rollback-test- (testing "Downgrading past the changeset (the real rollback path - the target version has none of the new- sample database code) removes the SQLite sample database and the whole Example collection tree that- the CreateSampleContent migration installed"- ;; Start early enough that CreateSampleContent runs under our binding and installs the real SQLite Sample- ;; Database and Example collection tree - the exact state a SQLite-sample version leaves to be torn down.- (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]- (binding [custom-migrations/*create-sample-content* true]- (migrate!))- (let [sample-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))- sample-coll-ids (mapv :id (t2/query {:select [:id] :from [:collection] :where [:= :is_sample true]}))]- (testing "precondition: the SQLite sample database and its Example collections exist after upgrade"- (is (some? sample-db-id))- (is (seq sample-coll-ids)))- (migrate! :down 62)- (testing "the SQLite sample database is gone"- (is (not (t2/exists? (t2/table-name :model/Database) :id sample-db-id))))- (testing "the entire outgoing Example collection tree is deleted - the cascade is complete"- (is (zero? (t2/count (t2/table-name :model/Collection) :id [:in sample-coll-ids]))))))))--(deftest migrate-away-from-sqlite-sample-database-on-downgrade-test- (testing "Downgrade removes the SQLite sample database and the content it leaves empty, keeping everything else"- (mt/with-temp- [:model/Database sample {:engine :sqlite, :is_sample true, :details {:db "mem:sample"}}- :model/Database other {:engine :h2, :details {:db "mem:other"}}- :model/Card other-card {:database_id (:id other)}- :model/Dashboard sample-dash {}- :model/Dashboard mixed-dash {}- :model/Collection examples {:name "Examples", :is_sample true}- :model/Collection ecommerce {:name "E-commerce", :is_sample true, :location (str "/" (:id examples) "/")}- :model/Collection keep-coll {:name "Keep me"}]- ;; Mirror the bundled sample content's fan-out: 8 tables x 7 fields, 39 cards, plus dashcards,- ;; series, and tabs. MySQL 9.7 resolves a multi-level ON DELETE CASCADE of this shape- ;; incompletely (it deletes the tables but orphans most cards and all fields), so the cleanup- ;; must not rely on DB-level cascade. This fixture makes that regression visible.- (let [table-ids (vec (t2/insert-returning-pks! :model/Table- (for [i (range 8)]- {:db_id (:id sample)- :name (str "sample_table_" i)- :active true})))- _ (t2/insert! :model/Field- (for [tid table-ids- i (range 7)]- {:table_id tid- :name (str "field_" i)- :base_type :type/Text- :database_type "TEXT"- :position i}))- card-ids (vec (t2/insert-returning-pks! :model/Card- (for [i (range 39)]- {:name (str "sample card " i)- :display "table"- :dataset_query {}- :visualization_settings {}- :creator_id (mt/user->id :rasta)- :database_id (:id sample)- :table_id (nth table-ids (mod i (count table-ids)))})))- [dc1 _dc2 dc3] (t2/insert-returning-pks! :model/DashboardCard- [{:dashboard_id (:id sample-dash) :card_id (first card-ids)- :size_x 4 :size_y 4 :row 0 :col 0}- {:dashboard_id (:id mixed-dash) :card_id (second card-ids)- :size_x 4 :size_y 4 :row 0 :col 0}- {:dashboard_id (:id mixed-dash) :card_id (:id other-card)- :size_x 4 :size_y 4 :row 0 :col 4}])- _ (t2/insert! :model/DashboardCardSeries {:dashboardcard_id dc1- :card_id (nth card-ids 2)- :position 0})- _ (t2/insert! :model/DashboardTab {:dashboard_id (:id sample-dash)- :name "Tab 1"- :position 0})- ;; parameter_card rows orphan in three ways; a fourth must survive.- [pc-src- pc-card-owner- pc-dash-owner- pc-keep] (t2/insert-returning-pks!- :model/ParameterCard- [;; value source is a sample card -> delete- {:card_id (first card-ids) :parameterized_object_type "dashboard"- :parameterized_object_id (:id mixed-dash) :parameter_id "src"}- ;; parameterized object is a sample card -> delete- {:card_id (:id other-card) :parameterized_object_type "card"- :parameterized_object_id (second card-ids) :parameter_id "card-owner"}- ;; parameterized object is a deleted sample dashboard -> delete- {:card_id (:id other-card) :parameterized_object_type "dashboard"- :parameterized_object_id (:id sample-dash) :parameter_id "dash-owner"}- ;; non-sample source on a surviving dashboard -> keep- {:card_id (:id other-card) :parameterized_object_type "dashboard"- :parameterized_object_id (:id mixed-dash) :parameter_id "keep"}])]- ;; Collection creation auto-grants an "All Users" permission row (object "/collection/<id>/") for each- ;; sample collection; the migration must clean those up too.- (#'custom-migrations/remove-sqlite-sample-database-on-downgrade!)- (testing "the SQLite sample database and all of its child content are deleted, with no orphans"- (is (not (t2/exists? :model/Database :id (:id sample))))- (is (zero? (t2/count :metabase_table :db_id (:id sample))))- (is (zero? (t2/count :metabase_field :table_id [:in table-ids])))- (is (zero? (t2/count :report_card :database_id (:id sample))))- (is (zero? (t2/count :report_dashboardcard :card_id [:in card-ids])))- (is (zero? (t2/count :dashboardcard_series :card_id [:in card-ids]))))- (testing "a dashboard left empty by the deletion is deleted, along with its tabs"- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))- (is (zero? (t2/count :dashboard_tab :dashboard_id (:id sample-dash)))))- (testing "parameter_card rows referencing a deleted sample card or dashboard are removed"- (is (not (t2/exists? :model/ParameterCard :id pc-src)))- (is (not (t2/exists? :model/ParameterCard :id pc-card-owner)))- (is (not (t2/exists? :model/ParameterCard :id pc-dash-owner)))- (testing "but an unrelated parameter_card is kept"- (is (t2/exists? :model/ParameterCard :id pc-keep))))- (testing "the sample Example collections and their permission records are deleted"- (is (not (t2/exists? :model/Collection :id (:id examples))))- (is (not (t2/exists? :model/Collection :id (:id ecommerce))))- (is (zero? (t2/count :permissions :object [:in [(format "/collection/%d/" (:id examples))- (format "/collection/%d/read/" (:id examples))]]))))- (testing "a dashboard that still has other cards, and unrelated content, is kept"- (is (t2/exists? :model/Dashboard :id (:id mixed-dash)))- (is (t2/exists? :report_dashboardcard :id dc3))- (is (t2/exists? :model/Card :id (:id other-card)))- (is (t2/exists? :model/Database :id (:id other)))- (is (t2/exists? :model/Collection :id (:id keep-coll))))))))--(deftest ^:mb/old-migrations-test downgrade-keeps-user-content-and-deletes-sample-dependents-test- (testing "Downgrade deletes everything that depends on the sample DB (transitively, even user-created), then keeps- any Example collection a user still has their own content in - leaving the RESTRICT FKs intact rather than- aborting on them"- (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]- (binding [custom-migrations/*create-sample-content* true]- (migrate!))- (let [examples-id (:id (t2/query-one {:select [:id] :from [:collection]- :where [:= :is_sample true] :order-by [[:id :asc]]}))- ecommerce-id (:id (t2/query-one {:select [:id] :from [:collection]- :where [:= :is_sample true] :order-by [[:id :desc]]}))- sqlite-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))- sample-card (:id (t2/query-one {:select [:id] :from [:report_card] :where [:= :database_id sqlite-db-id] :limit 1}))- ins (fn [t r] (first (t2/insert-returning-pks! t r)))- other-db (ins :metabase_database {:name "User DB" :engine "h2" :is_sample false- :details "{}" :created_at :%now :updated_at :%now})- ;; a user question that doesn't touch the sample DB, filed in Examples -> must survive- user-card (ins :report_card {:name "keep me" :display "table" :dataset_query "{}"- :visualization_settings "{}" :creator_id 13371338 :database_id other-db- :collection_id examples-id :created_at :%now :updated_at :%now})- ;; a user model built on a sample card -> depends transitively -> must be deleted- user-model (ins :report_card {:name "built on sample" :display "table" :dataset_query "{}"- :visualization_settings "{}" :creator_id 13371338 :database_id other-db- :source_card_id sample-card :collection_id examples-id- :created_at :%now :updated_at :%now})- ;; a curated table + a transform filed in E-commerce (RESTRICT FKs) -> keep that collection, FKs intact- table-id (ins :metabase_table {:name "curated" :active true :db_id other-db- :collection_id ecommerce-id :created_at :%now :updated_at :%now})- transform-id (ins :transform {:name "tf" :source "{}" :target "{}" :source_type "query"- :collection_id ecommerce-id :created_at :%now :updated_at :%now})- exists? (fn [t id] (boolean (seq (t2/query {:select [1] :from [t] :where [:= :id id] :limit 1}))))- coll-of (fn [t id] (:collection_id (t2/query-one {:select [:collection_id] :from [t] :where [:= :id id]})))]- (migrate! :down 62)- (testing "the sample database and everything depending on it - including the user model - are deleted"- (is (not (exists? :metabase_database sqlite-db-id)))- (is (not (exists? :report_card sample-card)))- (is (not (exists? :report_card user-model))))- (testing "a user question that does not depend on the sample DB survives"- (is (exists? :report_card user-card)))- (testing "Example collections holding surviving user content are kept"- (is (exists? :collection examples-id))- (is (exists? :collection ecommerce-id)))- (testing "the RESTRICT-FK table and transform survive with their collection_id intact (no abort, no null-out)"- (is (exists? :metabase_table table-id))- (is (= ecommerce-id (coll-of :metabase_table table-id)))- (is (exists? :transform transform-id))- (is (= ecommerce-id (coll-of :transform transform-id))))))))--(deftest ^:mb/old-migrations-test downgrade-leaves-no-orphaned-card-children-test- ;; B1 characterization: the migration hand-deletes only the dashcard tables and lets DB-level ON DELETE CASCADE- ;; clear report_card's other ~11 children (query_field, query_table, card_bookmark, report_cardfavorite, ...).- ;; Its own comment warns that MySQL 9.7 resolves cascades incompletely. This test seeds those children for a- ;; sample card and asserts none survive the card's deletion - i.e. it FAILS if the relied-on cascade leaves- ;; orphans. It deliberately asserts only on the outcome (no orphans), not on how the cleanup is done.- (testing "downgrade leaves no child rows pointing at a deleted sample card"- (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]- (binding [custom-migrations/*create-sample-content* true]- (migrate!))- (let [internal-user 13371338- sqlite-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))- sample-card (:id (t2/query-one {:select [:id] :from [:report_card] :where [:= :database_id sqlite-db-id] :limit 1}))- ins (fn [t r] (first (t2/insert-returning-pks! t r)))]- ;; children of report_card that the migration leaves to ON DELETE CASCADE (not hand-deleted)- (ins :query_field {:card_id sample-card :column "TOTAL" :explicit_reference true})- (ins :query_table {:card_id sample-card :table "ORDERS"})- (ins :card_bookmark {:user_id internal-user :card_id sample-card})- (ins :report_cardfavorite {:card_id sample-card :owner_id internal-user :created_at :%now :updated_at :%now})- (#'custom-migrations/remove-sqlite-sample-database-on-downgrade!)- (testing "the sample card was deleted"- (is (not (t2/exists? :report_card :id sample-card))))- (testing "no child table is left with a row pointing at the deleted card"- (doseq [table [:query_field :query_table :card_bookmark :report_cardfavorite]]- (is (zero? (t2/count table :card_id sample-card))- (str table " has an orphaned row referencing the deleted sample card"))))))))diff --git a/test/metabase/sample_data/example_content_test.clj b/test/metabase/sample_data/example_content_test.cljdeleted file mode 100644index d8bd0060301..00000000000--- a/test/metabase/sample_data/example_content_test.clj+++ /dev/null@@ -1,141 +0,0 @@-(ns metabase.sample-data.example-content-test- (:require- [clojure.test :refer :all]- [clojure.walk :as walk]- [metabase.query-processor :as qp]- [metabase.sample-data.example-content :as example-content]- [metabase.sample-data.impl]- [metabase.sync.core :as sync]- [metabase.test :as mt]- [metabase.util.json :as json]- [toucan2.core :as t2]))--(deftest remap-blob-parameters-preserves-order-test- (testing "remap-blob keeps a card/dashboard's parameters in their designed order rather than re-sorting by :id"- (let [blob (json/encode [{:id "zzzz" :name "Date Range" :type "string/="}- {:id "aaaa" :name "Vendor" :type "string/="}])- out (json/decode+kw (#'example-content/remap-blob {} :parameters blob))]- (is (= ["Date Range" "Vendor"] (mapv :name out))- "parameters must stay in original order, not be alphabetized by :id"))))--(deftest remap-collection-location-test- (let [remap #'example-content/remap-collection-location]- (testing "root location is unchanged"- (is (= "/" (remap {2 20} "/"))))- (testing "each path segment is remapped independently"- (is (= "/20/" (remap {2 20} "/2/")))- (is (= "/20/30/" (remap {2 20, 3 30} "/2/3/"))))- (testing "a new id colliding with a still-pending old id does not double-substitute"- ;; old reduce-of-str/replace would turn \"/5/12/\" into \"/99/99/\"; per-segment remapping keeps them distinct- (is (= "/12/99/" (remap {5 12, 12 99} "/5/12/"))))- (testing "segments without a mapping keep their original id"- (is (= "/2/7/" (remap {2 2} "/2/7/"))))))--(defn- sample-database-db []- {:details (#'metabase.sample-data.impl/try-to-extract-sample-database! :sqlite)- :engine :sqlite- :name "Sample Database"})--(defn- field-ids-in-query- "All numeric field ids referenced anywhere in a (pMBQL) dataset_query."- [query]- (let [ids (volatile! #{})]- (walk/postwalk- (fn [x]- (when (and (vector? x) (= :field (first x)))- (let [id (last x)]- (when (number? id) (vswap! ids conj id))))- x)- query)- @ids))--(defn- table-ids-in-query- "All numeric :source-table ids referenced anywhere in a (pMBQL) dataset_query."- [query]- (let [ids (volatile! #{})]- (walk/postwalk- (fn [x]- (when (and (map? x) (number? (:source-table x)))- (vswap! ids conj (:source-table x)))- x)- query)- @ids))--(deftest example-card-queries-run-after-id-bumping-upgrade-test- (testing "After an upgrade that bumped app-db ids past the EDN-hardcoded range, every recreated example card runs.- Cards that build on another example card carry a `\"card__N\"` :source-table; export-mbql only remaps a- *numeric* :source-table, so these stay pinned to the EDN card id and now point at an unrelated/missing card."- (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard- :model/Dimension :model/Permissions]- ;; Bump table/field ids past the EDN range (8 tables, 71 fields).- (mt/with-temp [:model/Database other {}- :model/Table ot {:db_id (:id other)}]- (doseq [i (range 200)]- (t2/insert! :model/Field {:table_id (:id ot) :name (str "filler_" i) :base_type :type/Integer- :database_type "INT" :position i}))- ;; Bump card ids past the EDN range (39 cards) so the EDN card ids (e.g. card__1) are taken by these- ;; unrelated filler cards rather than by the recreated example cards.- (dotimes [_ 60]- (t2/insert! :model/Card (mt/with-temp-defaults :model/Card)))- (mt/with-temp [:model/Database db (sample-database-db)]- (sync/sync-database! db)- (example-content/recreate-example-content! (:id db))- (let [cards (->> (t2/select :model/Card :database_id (:id db))- (filter (comp #{:question :model :metric} :type))- (sort-by :id))]- (is (seq cards) "example cards were recreated")- (doseq [card cards]- (testing (format "%s %s (card %s)" (:type card) (pr-str (:name card)) (:id card))- (let [result (try (qp/process-query (:dataset_query card))- (catch Throwable e {:error (ex-message e)}))]- (is (= nil (:error result))- (format "query failed: %s" (:error result))))))))))))--(deftest recreate-example-content-test- (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard- :model/Dimension :model/Permissions]- ;; Pre-create an unrelated database with tables/fields so the sample DB's synced table/field ids do- ;; NOT coincide with the EDN ids - otherwise the remap would be an identity no-op and prove nothing.- (mt/with-temp [:model/Database other {}- :model/Table ot {:db_id (:id other)}]- (doseq [i (range 120)]- (t2/insert! :model/Field {:table_id (:id ot) :name (str "filler_" i) :base_type :type/Integer- :database_type "INT" :position i}))- (mt/with-temp [:model/Database db (sample-database-db)]- (sync/sync-database! db)- (example-content/recreate-example-content! (:id db))- (let [cards (t2/select :model/Card :database_id (:id db))- coll-ids (set (keep :collection_id cards))- valid-fid (set (t2/select-fn-set :id :model/Field- {:join [[:metabase_table :t] [:= :t.id :metabase_field.table_id]]- :where [:= :t.db_id (:id db)]}))- valid-tid (set (t2/select-fn-set :id :model/Table :db_id (:id db)))]- (testing "the example cards are recreated against the new sample database"- (is (= 39 (count cards)))- (is (every? #(= (:id db) (:database_id %)) cards)))- (testing "every card lives in the example collection"- (is (seq coll-ids))- (is (every? :is_sample (t2/select :model/Collection :id [:in coll-ids]))))- (testing "every field/table reference in every card resolves to a real column of the new database"- (doseq [card cards]- (let [q (:dataset_query card)]- (doseq [fid (field-ids-in-query q)]- (is (contains? valid-fid fid)- (format "card %s references missing field %s" (:name card) fid)))- (doseq [st (table-ids-in-query q)]- (is (contains? valid-tid st)- (format "card %s references missing table %s" (:name card) st))))))- (testing "the dashboard and its dashcards are recreated and resolve to recreated cards"- (let [dash (t2/select-one :model/Dashboard :collection_id [:in coll-ids])- dashcards (t2/select :model/DashboardCard :dashboard_id (:id dash))- card-ids (set (map :id cards))]- (is (some? dash))- (is (pos? (count dashcards)))- (is (every? #(or (nil? (:card_id %)) (contains? card-ids (:card_id %))) dashcards))))- (testing "a recreated card actually runs against the new sample database"- (let [runnable (first (filter #(and (nil? (:source_card_id %))- (= :question (:type %))- (seq (table-ids-in-query (:dataset_query %))))- cards))]- (is (some? runnable))- (is (pos? (count (mt/rows (qp/process-query (:dataset_query runnable)))))))))))))diff --git a/test/metabase/sample_data/impl_test.clj b/test/metabase/sample_data/impl_test.cljindex e2fdae0429c..e5a0924c2dc 100644--- a/test/metabase/sample_data/impl_test.clj+++ b/test/metabase/sample_data/impl_test.clj@@ -9,13 +9,12 @@[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.permissions.core :as perms][metabase.plugins.impl :as plugins]- [metabase.sample-data.example-content :as example-content]+ [metabase.query-processor :as qp][metabase.sample-data.impl :as sample-data][metabase.sync.core :as sync][metabase.sync.task.sync-databases-test :as task.sync-databases-test][metabase.test :as mt][metabase.util :as u]- [metabase.util.json :as json][metabase.warehouse-schema.models.field-values :as field-values][metabase.warehouses-rest.api-test :as api.database-test][toucan2.core :as t2])@@ -54,6 +53,79 @@;;; ----------------------------------------------------- Tests ------------------------------------------------------+(deftest migrate-sample-database-engine-in-place-test+ (testing "Upgrade path: migrating the sample DB from H2 to SQLite in place keeps every Database/Table/Field+ id, so content referencing those ids survives with no remapping and still queries correctly."+ (mt/with-model-cleanup [:model/Database :model/Card]+ ;; Install the pre-upgrade (v62-shape) H2 sample database and a user question that references a field.+ (let [h2-db (t2/insert-returning-instance! :model/Database+ {:name "Sample Database" :engine :h2 :is_sample true+ :details (#'sample-data/try-to-extract-sample-database! :h2)})]+ (sync/sync-database! h2-db)+ (let [orders-id (t2/select-one-pk :model/Table :db_id (:id h2-db) :name "ORDERS")+ total-id (t2/select-one-pk :model/Field :table_id orders-id :name "TOTAL")+ user-card (t2/insert-returning-instance! :model/Card+ {:name "user q" :database_id (:id h2-db) :table_id orders-id+ :display "scalar" :visualization_settings {} :creator_id (mt/user->id :rasta)+ :dataset_query {:database (:id h2-db) :type :query+ :query {:source-table orders-id+ :aggregation [[:sum [:field total-id nil]]]}}})+ before-tables (t2/select-fn-set :id :model/Table :db_id (:id h2-db))+ before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables])]+ (is (= #{"PUBLIC"} (t2/select-fn-set :schema :model/Table :db_id (:id h2-db)))+ "precondition: H2 tables live in the PUBLIC schema")+ ;; ---- the migration under test ----+ (#'sample-data/migrate-sample-database-engine-in-place! :sqlite (t2/select-one :model/Database :id (:id h2-db)))+ (testing "the database record is now SQLite"+ (is (= :sqlite (:engine (t2/select-one :model/Database :id (:id h2-db))))))+ (testing "the same tables remain (no new ones), now with schema = nil"+ (is (= before-tables (t2/select-fn-set :id :model/Table :db_id (:id h2-db))))+ (is (= #{nil} (t2/select-fn-set :schema :model/Table :db_id (:id h2-db)))))+ (testing "the same fields remain (ids preserved), so embedded query refs stay valid"+ (is (= before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables]))))+ (testing "the user card survives with its id and still queries the (now SQLite) sample DB"+ (is (t2/exists? :model/Card :id (:id user-card)))+ (let [result (qp/process-query (:dataset_query (t2/select-one :model/Card :id (:id user-card))))]+ (is (= :completed (:status result)))+ (is (pos? (count (mt/rows result)))))))))))++(deftest migrate-sample-database-engine-in-place-downgrade-test+ (testing "Downgrade path: migrating the sample DB from SQLite back to H2 in place keeps every id, so+ sample and user content survive with no remapping and still query correctly."+ (mt/with-model-cleanup [:model/Database :model/Collection :model/Card :model/Dashboard]+ ;; Install the SQLite sample database (the state a newer version leaves behind) and a user question+ ;; that references a field.+ (let [sqlite-db (t2/insert-returning-instance! :model/Database+ {:name "Sample Database" :engine :sqlite :is_sample true+ :details (#'sample-data/try-to-extract-sample-database! :sqlite)})]+ (sync/sync-database! sqlite-db)+ (let [orders-id (t2/select-one-pk :model/Table :db_id (:id sqlite-db) :name "ORDERS")+ total-id (t2/select-one-pk :model/Field :table_id orders-id :name "TOTAL")+ user-card (t2/insert-returning-instance! :model/Card+ {:name "user q" :database_id (:id sqlite-db) :table_id orders-id+ :display "scalar" :visualization_settings {} :creator_id (mt/user->id :rasta)+ :dataset_query {:database (:id sqlite-db) :type :query+ :query {:source-table orders-id+ :aggregation [[:sum [:field total-id nil]]]}}})+ before-tables (t2/select-fn-set :id :model/Table :db_id (:id sqlite-db))+ before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables])]+ (is (= #{nil} (t2/select-fn-set :schema :model/Table :db_id (:id sqlite-db)))+ "precondition: SQLite tables have a nil schema")+ ;; ---- the migration under test ----+ (#'sample-data/migrate-sample-database-engine-in-place! :h2 (t2/select-one :model/Database :id (:id sqlite-db)))+ (testing "the database record is now H2"+ (is (= :h2 (:engine (t2/select-one :model/Database :id (:id sqlite-db))))))+ (testing "the same tables remain (no new ones), now with schema = PUBLIC"+ (is (= before-tables (t2/select-fn-set :id :model/Table :db_id (:id sqlite-db))))+ (is (= #{"PUBLIC"} (t2/select-fn-set :schema :model/Table :db_id (:id sqlite-db)))))+ (testing "the same fields remain (ids preserved)"+ (is (= before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables]))))+ (testing "the user card survives with its id and still queries the (now H2) sample DB"+ (is (t2/exists? :model/Card :id (:id user-card)))+ (let [result (qp/process-query (:dataset_query (t2/select-one :model/Card :id (:id user-card))))]+ (is (= :completed (:status result)))+ (is (pos? (count (mt/rows result)))))))))))+(def ^:private extracted-db-path-regex #".*plugins/sample-database\.sqlite$")(deftest extract-sample-database-test@@ -132,142 +204,6 @@(is (re-matches extracted-db-path-regex(get-in (t2/select-one :model/Database :id (:id db)) [:details :db]))))))))-(deftest replace-sample-database-on-engine-change-test- (testing "When the bundled engine changed (H2 -> SQLite), the old sample DB and its dependent content are removed"- (mt/with-temp- [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}- :model/Database other-db {:engine :h2, :details {:db "mem:other"}}- :model/Card sample-card {:database_id (:id old-sample)}- :model/Card other-card {:database_id (:id other-db)}- :model/Dashboard sample-dash {}- :model/Dashboard mixed-dash {}- :model/DashboardCard _ {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}- :model/DashboardCard _ {:dashboard_id (:id mixed-dash), :card_id (:id sample-card)}- :model/DashboardCard _ {:dashboard_id (:id mixed-dash), :card_id (:id other-card)}- :model/Collection examples {:name "Examples", :is_sample true}- :model/Collection ecommerce {:name "E-commerce", :is_sample true, :location (str "/" (:id examples) "/")}- :model/Collection keep-coll {:name "Keep me"}]- (let [bundled-engine (#'sample-data/sample-database-engine)- synced-db-ids (atom [])]- ;; recreate (collection reuse) is exercised separately; here we only check the database swap + cleanup.- (with-redefs [example-content/recreate-example-content! (constantly nil)]- (mt/with-dynamic-fn-redefs [sync/sync-database! (fn [db] (swap! synced-db-ids conj (:id db)) db)]- (#'sample-data/update-sample-database-if-needed! old-sample)))- (testing "the old sample DB is deleted, cascading to its cards"- (is (not (t2/exists? :model/Database :id (:id old-sample))))- (is (not (t2/exists? :model/Card :id (:id sample-card)))))- (testing "a dashboard emptied by the deletion is deleted"- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash)))))- (testing "a dashboard that still has other cards is kept"- (is (t2/exists? :model/Dashboard :id (:id mixed-dash)))- (is (t2/exists? :model/Card :id (:id other-card))))- (testing "the Example collections are preserved (reused on recreate), not deleted by the engine swap"- (is (t2/exists? :model/Collection :id (:id examples)))- (is (t2/exists? :model/Collection :id (:id ecommerce)))- (is (t2/exists? :model/Collection :id (:id keep-coll))))- (testing "a new sample database with the bundled engine is created and synced"- ;; replace-sample-database! inserts the new sample DB outside any with-temp, so key off the synced id- ;; (authoritative) rather than (select-one :is_sample true), which would see leaks from other tests.- (is (= 1 (count @synced-db-ids)) "the new database (not the old one) is what gets synced")- (let [new-db (t2/select-one :model/Database :id (first @synced-db-ids))]- (is (some? new-db))- (is (not= (:id old-sample) (:id new-db)))- (is (= bundled-engine (:engine new-db)))- ;; clean it up so the leaked is_sample row can't pollute tests that select on is_sample.- (t2/delete! :model/Database :id (:id new-db))))))))--(deftest replace-sample-database-cascades-to-synced-schema-test- (testing "On H2 -> SQLite replacement, deleting the old sample DB also removes its synced schema (Tables, Fields,- Dimensions) and dashboard tabs - i.e. every entity belonging to the old H2 sample database is gone"- (mt/with-temp- [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}- :model/Table table {:db_id (:id old-sample)}- :model/Field field {:table_id (:id table)}- :model/Dimension dimension {:field_id (:id field)}- :model/Card sample-card {:database_id (:id old-sample)}- :model/Dashboard sample-dash {}- :model/DashboardTab tab {:dashboard_id (:id sample-dash)}- :model/DashboardCard _ {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}- :model/Collection examples {:name "Examples", :is_sample true}]- ;; Skip recreating the replacement Sample Database (load-sample-content? false): this test only checks the- ;; old schema cascades away, and recreating it would leak an is_sample row other tests select on.- (with-redefs [config/load-sample-content? (constantly false)]- (#'sample-data/update-sample-database-if-needed! old-sample))- (testing "the old sample DB is deleted"- (is (not (t2/exists? :model/Database :id (:id old-sample)))))- (testing "its synced schema cascades away"- (is (not (t2/exists? :model/Table :id (:id table))))- (is (not (t2/exists? :model/Field :id (:id field))))- (is (not (t2/exists? :model/Dimension :id (:id dimension)))))- (testing "its card and the dashboard emptied by the card's removal are deleted, along with the dashboard's tabs"- (is (not (t2/exists? :model/Card :id (:id sample-card))))- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))- (is (not (t2/exists? :model/DashboardTab :id (:id tab)))))- (testing "the Example collection is preserved (the engine swap no longer deletes it)"- (is (t2/exists? :model/Collection :id (:id examples)))))))--(deftest replace-sample-database-survives-dangling-visualizer-ref-test- (testing "The engine swap completes when a sample dashcard has a dangling visualizer ref"- (mt/with-temp- [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}- :model/Card sample-card {:database_id (:id old-sample)}- :model/Dashboard sample-dash {}- :model/DashboardCard dc {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}]- ;; Inject a dangling visualizer ref via raw SQL so the model's hooks don't reject/rewrite it.- (t2/query-one {:update (t2/table-name :model/DashboardCard)- :set {:visualization_settings- (json/encode {:visualization- {:columnValuesMapping- {:COLUMN_1 [{:sourceId "card:gEnfWx10SmfjiccZpcGrj"}]}}})}- :where [:= :id (:id dc)]})- (with-redefs [config/load-sample-content? (constantly false)]- (testing "the replacement completes instead of throwing on the dangling ref"- (is (nil? (#'sample-data/update-sample-database-if-needed! old-sample)))))- (testing "the old sample DB and the dashboard emptied by its removal are deleted"- (is (not (t2/exists? :model/Database :id (:id old-sample))))- (is (not (t2/exists? :model/Card :id (:id sample-card))))- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))))))--(def ^:private edn-example-collection-entity-id- "Stable entity id of the bundled Examples collection in sample-content.edn."- "HyB3nRtqb7pBPhFG26evI")--(deftest recreate-example-content-reuses-collection-test- (testing "recreate-example-content! reuses an existing Example collection (matched by entity id) and preserves the- user content filed into it, instead of deleting it and creating a brand new collection"- (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard- :model/Dimension :model/Permissions]- (with-temp-sample-database-db [db]- (mt/with-temp- [:model/Collection examples {:name "Examples", :is_sample true, :location "/"- :entity_id edn-example-collection-entity-id}- ;; a question a user filed into the Example collection - must survive the engine swap- :model/Card user-card {:name "user question", :collection_id (:id examples), :database_id (:id db)}]- (example-content/recreate-example-content! (:id db))- (testing "the existing Example collection is reused, not duplicated"- (is (t2/exists? :model/Collection :id (:id examples)))- (is (= 1 (t2/count :model/Collection :entity_id edn-example-collection-entity-id))))- (testing "the user's content survives, still filed in the reused Example collection"- (is (t2/exists? :model/Card :id (:id user-card)))- (is (= (:id examples) (:collection_id (t2/select-one :model/Card :id (:id user-card))))))- (testing "the bundled sample content is recreated on the sample database, in the reused collection"- (is (pos? (t2/count :model/Card :database_id (:id db))))- (is (t2/exists? :model/Dashboard :collection_id (:id examples)))))))))--(deftest replace-sample-database-skips-content-when-sync-fails-test- (testing "When the replacement sync fails, example content is NOT recreated - recreating against an unsynced DB- remaps every card onto empty id maps, which is silent corruption worse than a missing example collection"- (mt/with-model-cleanup [:model/Database]- (mt/with-temp [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}]- (let [recreate-called? (atom false)]- (with-redefs [example-content/recreate-example-content! (fn [_] (reset! recreate-called? true))]- (mt/with-dynamic-fn-redefs [sync/sync-database! (fn [_] (throw (ex-info "sync boom" {})))]- (#'sample-data/update-sample-database-if-needed! old-sample)))- (testing "recreate is skipped"- (is (false? @recreate-called?)))- (testing "the engine swap itself still completed - a new SQLite sample DB exists"- (is (t2/exists? :model/Database :is_sample true :engine :sqlite))))))))-(defn- db-level-perms"DB-level data-permission rows for `db-id` as a comparable map {[group-id perm-type] perm-value}."[db-id]diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.cljindex 88f09d1a2d3..f1893bc8d9d 100644--- a/test/metabase/warehouses/models/database_test.clj+++ b/test/metabase/warehouses/models/database_test.clj@@ -708,20 +708,6 @@(testing ":write_data_details :auth-provider is keywordized"(is (keyword? (get-in db [:write_data_details :auth-provider]))))))))-(deftest user-may-not-update-sample-database-test- (mt/with-temp [:model/Database {:keys [id] :as _sample-database} {:engine :h2- :is_sample true- :name "Sample Database"- :details {:db "./resources/sample-database.db;USER=GUEST;PASSWORD=guest"}}]- (testing " updating the engine of a sample database is not allowed"- (is (thrown-with-msg?- clojure.lang.ExceptionInfo- #"The engine on a sample database cannot be changed."- (t2/update! :model/Database id {:engine :sqlite}))))- (testing " updating other attributes of a sample database is allowed"- (t2/update! :model/Database id {:name "My New Name"})- (is (= "My New Name" (t2/select-one-fn :name :model/Database :id id))))))-(driver/register! ::test, :abstract? true)(deftest preserve-driver-namespaces-test