Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Ghy-4069 minimal migration approach (#77297)

Ben Grabowf0ef171d2587
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 24cf1b6b213..51db75d23fb 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1947,19 +1947,11 @@
{:team "UX West"
:api #{metabase.sample-data.core
metabase.sample-data.init}
- :uses #{config
- models
- permissions
- plugins
+ :uses #{plugins
settings
sync
util}
- :model-imports #{:model/Card
- :model/Dashboard
- :model/DashboardCard
- :model/DataPermissions
- :model/Database
- :model/Field
+ :model-imports #{:model/Database
:model/Table}}
search
diff --git a/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml b/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml
index 603f5c154a1..3c975258f4b 100644
--- a/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml
+++ b/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml
@@ -1,12 +1,10 @@
-## On downgrade from a SQLite-sample-database version to an H2-sample-database version, remove the
-## SQLite sample database and the content left dangling by it. Does nothing on rollforward; the
-## H2 -> SQLite upgrade replacement is handled at startup in metabase.sample-data.impl.
+## No-op; superseded by the in-place sample-database engine swap at startup (metabase.sample-data.impl).
databaseChangeLog:
- changeSet:
id: v63.2026-06-08T00:00:00
author: bgrabow
- comment: Remove the SQLite sample database (and content it leaves empty) on downgrade to an H2 version
+ comment: No-op; sample database engine swap is handled at startup
validCheckSum: ANY
changes:
- customChange:
diff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.clj
index 2fdb9042497..545524031e0 100644
--- a/src/metabase/app_db/custom_migrations.clj
+++ b/src/metabase/app_db/custom_migrations.clj
@@ -1330,120 +1330,9 @@
(custom-migrations.util/with-temp-schedule! [scheduler]
(qs/delete-job scheduler (jobs/key "metabase.task.send-pulses.init-send-pulse-triggers.job"))))
-(defn- card-ids-depending-on-database
- "Set of report_card ids that depend on `database-id` directly (their own query targets it) or transitively
- (a card built on such a card via source_card_id, to any depth)."
- [database-id]
- (loop [frontier (into #{} (map :id) (t2/query {:select [:id] :from [:report_card]
- :where [:= :database_id database-id]}))
- acc #{}]
- (if (empty? frontier)
- acc
- (let [acc' (into acc frontier)
- next* (into #{} (map :id) (t2/query {:select [:id] :from [:report_card]
- :where [:and [:in :source_card_id frontier]
- [:not [:in :id acc']]]}))]
- (recur next* acc')))))
-
-(defn- collection-has-survivors?
- "True if `coll-id` still holds anything after sample content is gone: any content row filed into it, or any
- surviving child collection (whose path is exactly `child-location`). Checked bottom-up so a child that itself
- survived keeps its parent alive."
- [coll-id child-location]
- (boolean
- (or (seq (t2/query {:select [1] :from [:collection] :where [:= :location child-location] :limit 1}))
- (some (fn [table]
- (seq (t2/query {:select [1] :from [table] :where [:= :collection_id coll-id] :limit 1})))
- [:report_card :report_dashboard :native_query_snippet :pulse :timeline :document
- :metabase_table :transform]))))
-
-(defn- remove-sqlite-sample-database-on-downgrade!
- "Delete the SQLite sample database and everything that depends on it - its tables/fields, every Card whose
- query targets it (directly or transitively through source_card_id), those cards' dashcards, and Dashboards
- left empty as a result - even when that content was user-created, because it cannot work without the database.
-
- The Example collections are then pruned, bottom-up: a sample collection is deleted only if nothing is left in
- it once the sample content is gone (no content rows, no surviving child collection); a collection a user kept
- their own content in is left intact. Permission records for the deleted collections go with them.
-
- Children are deleted explicitly, bottom-up, rather than relying on ON DELETE CASCADE: MySQL 9.7 resolves
- multi-level cascade fan-outs incompletely, leaving orphaned rows that later break ALTER TABLE statements which
- re-validate foreign keys."
- []
- (when-let [sample-db-id (:id (t2/query-one {:select [:id]
- :from [:metabase_database]
- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))]
- (let [card-ids (card-ids-depending-on-database sample-db-id)
- field-ids-q {:select [:id]
- :from [:metabase_field]
- :where [:in :table_id {:select [:id]
- :from [:metabase_table]
- :where [:= :db_id sample-db-id]}]}
- affected-dashboard-ids (if (empty? card-ids)
- #{}
- (->> (t2/query {:select-distinct [:dashboard_id]
- :from [:report_dashboardcard]
- :where [:in :card_id card-ids]})
- (into #{} (map :dashboard_id))))]
- (when (seq card-ids)
- (t2/query {:delete-from :dashboardcard_series
- :where [:or
- [:in :card_id card-ids]
- [:in :dashboardcard_id {:select [:id]
- :from [:report_dashboardcard]
- :where [:in :card_id card-ids]}]]})
- (t2/query {:delete-from :report_dashboardcard :where [:in :card_id card-ids]})
- (t2/query {:delete-from :parameter_card
- :where [:or
- [:in :card_id card-ids]
- [:and [:= :parameterized_object_type "card"]
- [:in :parameterized_object_id card-ids]]]})
- (t2/query {:delete-from :report_card :where [:in :id card-ids]}))
- (t2/query {:delete-from :dimension :where [:in :field_id field-ids-q]})
- (t2/query {:delete-from :metabase_field
- :where [:in :table_id {:select [:id]
- :from [:metabase_table]
- :where [:= :db_id sample-db-id]}]})
- (t2/query {:delete-from :metabase_table :where [:= :db_id sample-db-id]})
- (t2/query {:delete-from :metabase_database :where [:= :id sample-db-id]})
- (when (seq affected-dashboard-ids)
- ;; A dashboard that depended on the sample DB is removed once no card-backed dashcard survives - even if
- ;; text/heading dashcards remain (the example dashboard is built entirely from sample cards plus headings).
- ;; A dashboard that still has a real card of its own (e.g. a user mixed in another card) is left alone.
- (let [non-empty (->> (t2/query {:select-distinct [:dashboard_id]
- :from [:report_dashboardcard]
- :where [:and [:in :dashboard_id affected-dashboard-ids]
- [:not= :card_id nil]]})
- (into #{} (map :dashboard_id)))
- empty-ids (remove non-empty affected-dashboard-ids)]
- (when (seq empty-ids)
- (t2/query {:delete-from :report_dashboardcard :where [:in :dashboard_id empty-ids]})
- (t2/query {:delete-from :dashboard_tab :where [:in :dashboard_id empty-ids]})
- (t2/query {:delete-from :parameter_card
- :where [:and [:= :parameterized_object_type "dashboard"]
- [:in :parameterized_object_id empty-ids]]})
- (t2/query {:delete-from :report_dashboard :where [:in :id empty-ids]}))))
- ;; Prune the Example collections deepest-first, so an emptied child is gone before its parent is judged.
- (doseq [{:keys [id location]} (->> (t2/query {:select [:id :location] :from [:collection] :where [:= :is_sample true]})
- (sort-by (comp count :location) >))]
- (when-not (collection-has-survivors? id (str location id "/"))
- (t2/query {:delete-from :permissions
- :where [:or
- [:= :collection_id id]
- [:in :object [(format "/collection/%d/" id)
- (format "/collection/%d/read/" id)]]]})
- (t2/query {:delete-from :collection :where [:= :id id]}))))))
-
-;; The bundled sample database moved from H2 to SQLite. If an instance running a SQLite-sample
-;; version is downgraded to an H2-sample version, the older code cannot use the SQLite sample
-;; database (it has the wrong engine/details) and would leave it broken. On downgrade we therefore
-;; remove the SQLite sample database and the content left dangling by it, and we do NOT restore an
-;; H2 sample database. This mirrors the upgrade behavior in
-;; metabase.sample-data.impl/replace-sample-database!. There is no forward migration: the upgrade
-;; (H2 -> SQLite) replacement is handled at startup, not here.
-(define-reversible-migration MigrateAwayFromSqliteSampleDatabaseOnDowngrade
- (log/info "No forward migration for MigrateAwayFromSqliteSampleDatabaseOnDowngrade")
- (remove-sqlite-sample-database-on-downgrade!))
+;; No-op. The SQLite <-> H2 sample database engine swap is handled in place at startup
+;; (metabase.sample-data.impl). Kept because a shipped v63 build ran a delete-based rollback here.
+(define-reversible-migration MigrateAwayFromSqliteSampleDatabaseOnDowngrade nil nil)
;; when card display is area or bar,
;; 1. set the display key to :stackable.stack_display value OR leave it the same
diff --git a/src/metabase/sample_data/example_content.clj b/src/metabase/sample_data/example_content.clj
deleted file mode 100644
index 06e1fc65cfa..00000000000
--- a/src/metabase/sample_data/example_content.clj
+++ /dev/null
@@ -1,285 +0,0 @@
-(ns metabase.sample-data.example-content
- "Recreate the bundled Example collection (the content shipped in `sample-content.edn`) on top of an
- already-synced Sample Database.
-
- On a fresh install `metabase.app-db.custom-migrations/CreateSampleContentV2` inserts the EDN rows
- verbatim into empty tables, so auto-increment reproduces the EDN ids exactly and every embedded
- reference still lines up. On upgrade the tables are not empty, so we insert the same rows but remap
- each id: simple FK columns by lookup, and the Database/Table/Field/Card ids embedded in card and
- dashcard query blobs via the serdes export walkers bound to numeric id maps."
- (:require
- [clojure.edn :as edn]
- [clojure.java.io :as io]
- [clojure.string :as str]
- [clojure.walk :as walk]
- [medley.core :as m]
- [metabase.models.interface :as mi]
- [metabase.models.serialization :as serdes]
- [metabase.settings.core :as setting]
- [metabase.util.date-2 :as u.date]
- [metabase.util.json :as json]
- [metabase.util.log :as log]
- [toucan2.core :as t2]))
-
-(set! *warn-on-reflection* true)
-
-(def ^:private sample-content-resource "sample-content.edn")
-
-(defn- load-sample-content []
- (with-open [r (io/reader (io/resource sample-content-resource))]
- (edn/read {:readers {'t u.date/parse}} (java.io.PushbackReader. r))))
-
-(defn- now-for-temporals [row]
- (update-vals row (fn [v] (if (instance? java.time.temporal.Temporal v) :%now v))))
-
-(defn- prep-row
- "Strip the EDN id (let the app DB assign one) and replace temporal literals with `:%now`."
- [row]
- (-> row now-for-temporals (dissoc :id)))
-
-(defn- insert-rows!
- "Insert `rows` (each carrying its original EDN `:id` and `:entity_id`) into `table` after prepping,
- and return a map of `edn-id -> new id` so other rows' numeric FK references can be remapped. The new
- id is recovered by matching the unique `:entity_id` back to the inserted row."
- [table rows]
- (when (seq rows)
- (t2/query {:insert-into table :values (mapv prep-row rows)}))
- (let [eid->new (into {}
- (map (juxt :entity_id :id))
- (t2/query {:select [:id :entity_id] :from table
- :where [:in :entity_id (mapv :entity_id rows)]}))]
- (into {} (map (fn [r] [(:id r) (eid->new (:entity_id r))])) rows)))
-
-;;; --------------------------------------------- id remapping ----------------------------------------------
-
-(defn- model->map-key [model]
- (case (name model)
- "Card" :cards
- "Dashboard" :dashboards
- "Collection" :collections
- nil))
-
-(defn- remap-result-metadata [maps cols]
- (mapv (fn [m]
- (-> m
- (m/update-existing :table_id #(get (:tables maps) % %))
- (m/update-existing :id #(if (number? %) (get (:fields maps) % %) %))
- (m/update-existing :field_ref serdes/export-mbql)
- (m/update-existing :fk_target_field_id #(if (number? %) (get (:fields maps) % %) %))))
- cols))
-
-(defn- remap-source-card-refs
- "`export-mbql` only remaps a numeric `:source-table` (a table id); a `\"card__N\"` `:source-table` - a card
- that builds on another card - is left carrying the EDN card id. Rewrite each through the card id map so it
- points at the recreated card instead of whatever row happens to occupy the EDN id on an upgraded app db."
- [card-map query]
- (walk/postwalk
- (fn [x]
- (if-let [[_ n] (and (map? x)
- (string? (:source-table x))
- (re-matches #"card__(\d+)" (:source-table x)))]
- (let [old-id (parse-long n)]
- (assoc x :source-table (str "card__" (get card-map old-id old-id))))
- x))
- query))
-
-(defn- remap-blob
- "Remap the entity ids embedded in a serialized JSON `blob` string of the given `kind`, returning a
- JSON string. Reuses the serdes export walkers with the FK resolvers bound to return new numeric ids
- (rather than portable references)."
- [maps kind blob]
- (when (some? blob)
- (binding [serdes/*export-database-fk* (fn [_id] (:db maps))
- serdes/*export-field-fk* (fn [id] (get (:fields maps) id id))
- serdes/*export-table-fk* (fn [id] (get (:tables maps) id id))
- serdes/*export-fk* (fn [id model]
- (let [k (model->map-key model)]
- (if k (get (get maps k) id id) id)))]
- (json/encode
- (case kind
- :mbql (->> (json/decode+kw blob) serdes/export-mbql (remap-source-card-refs (:cards maps)))
- ;; Remap each parameter's refs in place, preserving the designed order. (export-parameters would reorder
- ;; them by :id - a serialization-stability device we must not inherit, or the filter order is scrambled.)
- :parameters (mapv serdes/export-mbql (json/decode+kw blob))
- :param-maps (serdes/export-parameter-mappings (json/decode+kw blob))
- ;; column_settings keys must stay strings, so decode without keywordizing (as the model does).
- :viz (serdes/export-visualization-settings (mi/normalize-visualization-settings (json/decode blob)))
- :result-md (remap-result-metadata maps (json/decode+kw blob)))))))
-
-;;; --------------------------------------------- insertion ----------------------------------------------
-
-(defn- field-id-maps
- "Build `{:db new-db-id :tables {edn-table-id new-table-id} :fields {edn-field-id new-field-id}}` by
- matching the EDN table/field names against the freshly-synced Sample Database."
- [content new-db-id]
- (let [edn-tables (:metabase_table content)
- edn-fields (:metabase_field content)
- tname->new (t2/select-fn->pk :name :model/Table :db_id new-db-id)
- table-id-map (into {} (keep (fn [{:keys [id name]}]
- (when-let [new-id (tname->new name)]
- [id new-id])))
- edn-tables)
- ;; [new-table-id field-name] -> new-field-id
- new-fields (t2/select :model/Field :table_id [:in (vals table-id-map)])
- nf-index (into {} (map (fn [f] [[(:table_id f) (:name f)] (:id f)])) new-fields)
- field-id-map (into {} (keep (fn [{:keys [id table_id name]}]
- (when-let [new-tid (table-id-map table_id)]
- (when-let [new-fid (nf-index [new-tid name])]
- [id new-fid])))
- edn-fields))]
- {:db new-db-id :tables table-id-map :fields field-id-map}))
-
-(defn- remap-collection-location
- "Remap the parent ids embedded in a collection `location` path (e.g. \"/2/3/\") through `old->new`, one path
- segment at a time. Remapping per segment (rather than successive string replacement over the id map) avoids a
- newly-assigned id being re-substituted as if it were a still-pending old id."
- [old->new location]
- (if (= "/" location)
- location
- (str "/"
- (->> (str/split location #"/")
- (remove str/blank?)
- (map (fn [segment]
- (if-let [old-id (parse-long segment)]
- (str (get old->new old-id old-id))
- segment)))
- (str/join "/"))
- "/")))
-
-(defn- insert-collections! [content]
- ;; Reuse the existing Example collections (matched by entity id) so user content filed into them survives an
- ;; engine swap; only create one when it's missing. Roots before children so a new child's location path can be
- ;; remapped against ids already assigned.
- (loop [pending (sort-by (comp count :location) (:collection content))
- coll-map {}]
- (if-let [row (first pending)]
- (let [existing (:id (t2/query-one {:select [:id] :from :collection :where [:= :entity_id (:entity_id row)]}))
- new-id (or existing
- (let [prepared (-> row prep-row (update :location #(remap-collection-location coll-map %)))]
- (t2/query {:insert-into :collection :values [prepared]})
- (:id (t2/query-one {:select [:id] :from :collection :where [:= :entity_id (:entity_id row)]}))))]
- (recur (rest pending) (assoc coll-map (:id row) new-id)))
- coll-map)))
-
-(defn- delete-prior-sample-content!
- "Drop the previously-bundled sample cards and dashboards, matched by their stable EDN entity ids, so reinserting
- them can't collide and stale ones don't linger. User content never carries these entity ids, so it is untouched;
- the Example collections are reused, not deleted. Dashboards are removed before cards because deleting a dashboard
- cascades its dashcards."
- [content]
- (doseq [table [:report_dashboard :report_card]]
- (let [eids (into [] (keep :entity_id) (get content table))]
- (when (seq eids)
- (t2/query {:delete-from table :where [:in :entity_id eids]})))))
-
-(defn- insert-cards! [content maps]
- ;; Phase 1: insert with simple FKs remapped and blobs left as-is (no FK enforcement on JSON), so all
- ;; card ids exist before we rewrite inter-card references in phase 2.
- (let [coll-map (:collections maps)
- rows (for [card (:report_card content)]
- (-> card
- (assoc :database_id (:db maps))
- (assoc :collection_id (some-> (:collection_id card) coll-map))
- (assoc :table_id (some-> (:table_id card) (->> (get (:tables maps)))))
- (assoc :source_card_id nil)))
- card-map (insert-rows! :report_card (vec rows))
- maps (assoc maps :cards card-map)]
- ;; Phase 2: rewrite query blobs + inter-card refs now that every card id exists.
- (doseq [card (:report_card content)]
- (t2/query {:update :report_card
- :set {:dataset_query (remap-blob maps :mbql (:dataset_query card))
- :result_metadata (remap-blob maps :result-md (:result_metadata card))
- :parameters (remap-blob maps :parameters (:parameters card))
- :parameter_mappings (remap-blob maps :param-maps (:parameter_mappings card))
- :visualization_settings (remap-blob maps :viz (:visualization_settings card))
- :source_card_id (some-> (:source_card_id card) card-map)}
- :where [:= :id (card-map (:id card))]}))
- maps))
-
-(defn- insert-dashboards! [content maps]
- (let [coll-map (:collections maps)
- rows (mapv (fn [d] (assoc d :collection_id (some-> (:collection_id d) coll-map)))
- (:report_dashboard content))
- dash-map (insert-rows! :report_dashboard rows)
- maps (assoc maps :dashboards dash-map)]
- (doseq [d (:report_dashboard content)]
- (t2/query {:update :report_dashboard
- :set {:parameters (remap-blob maps :parameters (:parameters d))}
- :where [:= :id (dash-map (:id d))]}))
- maps))
-
-(defn- insert-dashboard-tabs! [content maps]
- (let [dash-map (:dashboards maps)
- rows (mapv (fn [t] (assoc t :dashboard_id (dash-map (:dashboard_id t))))
- (:dashboard_tab content))]
- (assoc maps :tabs (insert-rows! :dashboard_tab rows))))
-
-(defn- insert-dashcards! [content maps]
- (let [{:keys [dashboards cards tabs]} maps
- rows (mapv (fn [dc]
- (-> (prep-row dc)
- (assoc :dashboard_id (dashboards (:dashboard_id dc)))
- (assoc :card_id (some-> (:card_id dc) cards))
- (assoc :dashboard_tab_id (some-> (:dashboard_tab_id dc) tabs))
- (assoc :parameter_mappings (remap-blob maps :param-maps (:parameter_mappings dc)))
- (assoc :visualization_settings (remap-blob maps :viz (:visualization_settings dc)))))
- (:report_dashboardcard content))]
- (when (seq rows)
- (t2/query {:insert-into :report_dashboardcard :values rows}))
- maps))
-
-(defn- insert-dimensions! [content maps]
- (let [fields (:fields maps)
- rows (mapv (fn [dim]
- (-> (prep-row dim)
- (assoc :field_id (fields (:field_id dim)))
- (m/update-existing :human_readable_field_id #(some->> % (get fields)))))
- (:dimension content))]
- (when (seq rows)
- (t2/query {:insert-into :dimension :values rows}))))
-
-(defn- grant-example-collection-perms! [content maps]
- (when-let [example-collection-id (some->> (:collection content)
- (filter :is_sample)
- first :id
- (get (:collections maps)))]
- (when-let [group-id (:id (t2/query-one {:select [:id] :from :permissions_group :where [:= :name "All Users"]}))]
- ;; A reused Example collection already carries this grant; only insert when it's missing.
- (when-not (seq (t2/query {:select [1] :from :permissions
- :where [:and [:= :collection_id example-collection-id]
- [:= :group_id group-id]
- [:= :perm_type "perms/collection-access"]]
- :limit 1}))
- (t2/query {:insert-into :permissions
- :values [{:object (format "/collection/%s/" example-collection-id)
- :group_id group-id
- :perm_type "perms/collection-access"
- :perm_value "read-and-write"
- :collection_id example-collection-id}]})))))
-
-(defn- set-example-dashboard-id! [content maps]
- (when-let [dash (first (:report_dashboard content))]
- (when-let [new-id (get (:dashboards maps) (:id dash))]
- (setting/set-value-of-type! :integer :example-dashboard-id new-id))))
-
-(defn recreate-example-content!
- "Recreate the bundled Example collection on the freshly-synced Sample Database `new-db-id`. Mirrors
- the fresh-install seeding, remapping every id reference to point at the new database's tables and
- fields. Best-effort: logs and swallows errors so a content-seeding failure can't break startup."
- [new-db-id]
- (try
- (t2/with-transaction [_conn]
- (let [content (load-sample-content)
- _ (delete-prior-sample-content! content)
- maps (field-id-maps content new-db-id)
- maps (assoc maps :collections (insert-collections! content))
- maps (insert-cards! content maps)
- maps (insert-dashboards! content maps)
- maps (insert-dashboard-tabs! content maps)
- maps (insert-dashcards! content maps)]
- (insert-dimensions! content maps)
- (grant-example-collection-perms! content maps)
- (set-example-dashboard-id! content maps)))
- (log/info "Recreated Sample Database example content")
- (catch Throwable e
- (log/error e "Failed to recreate Sample Database example content"))))
diff --git a/src/metabase/sample_data/impl.clj b/src/metabase/sample_data/impl.clj
index 678b61945be..1a3f229fca6 100644
--- a/src/metabase/sample_data/impl.clj
+++ b/src/metabase/sample_data/impl.clj
@@ -3,10 +3,7 @@
(:require
[clojure.java.io :as io]
[clojure.string :as str]
- [metabase.config.core :as config]
- [metabase.permissions.core :as perms]
[metabase.plugins.core :as plugins]
- [metabase.sample-data.example-content :as example-content]
[metabase.sync.core :as sync]
[metabase.util.files :as u.files]
[metabase.util.i18n :refer [trs]]
@@ -112,118 +109,34 @@
(catch Throwable e
(log/error e "Failed to load sample database"))))
-(defn- sample-database-dashboard-ids
- "IDs of Dashboards holding at least one card backed by `database-id`. Captured before the sample
- database is deleted so the dashboards it empties out can be pruned afterward."
- [database-id]
- (t2/select-fn-set :dashboard_id [:model/DashboardCard :dashboard_id]
- :card_id [:in {:select [:id]
- :from [(t2/table-name :model/Card)]
- :where [:= :database_id database-id]}]))
-
-(defn- delete-emptied-dashboards!
- "Delete whichever of `dashboard-ids` no longer have any dashcards. A sample dashboard whose cards
- were all removed with the sample database is deleted; a dashboard that still has cards (e.g. mixes
- in cards from another database) is left alone."
- [dashboard-ids]
- (when (seq dashboard-ids)
- (let [non-empty (t2/select-fn-set :dashboard_id [:model/DashboardCard :dashboard_id] :dashboard_id [:in dashboard-ids])
- empty-ids (remove (or non-empty #{}) dashboard-ids)]
- (when (seq empty-ids)
- (t2/delete! :model/Dashboard :id [:in empty-ids])))))
-
-(defn- capture-permissions-snapshot
- "Snapshot a database's data-permission rows plus the names of the tables referenced by any table-level
- rows, so the same access can be re-applied to a replacement database whose tables have new IDs. Must run
- before the old database (and its cascading permission rows) is deleted."
- [db-id]
- (let [permissions (t2/select :model/DataPermissions :db_id db-id)
- table-ids (into #{} (keep :table_id) permissions)]
- {:permissions permissions
- :table-id->name (if (seq table-ids)
- (t2/select-pk->fn :name :model/Table :id [:in table-ids])
- {})}))
-
-(defn- apply-permissions-snapshot!
- "Re-apply a permission snapshot (see [[capture-permissions-snapshot]]) to the replacement database
- `new-db-id`, replacing the defaults granted when it was created. Table-level rows are remapped onto the
- new database's tables by name; since the new tables exist only after the first sync, this must run
- post-sync. A table-level row whose table has no name match in the new database is dropped."
- [new-db-id {:keys [permissions table-id->name]}]
- (let [name->new-table-id (t2/select-fn->pk :name :model/Table :db_id new-db-id)
- granular? (fn [rows] (some :table_id rows))
- permission-groups (group-by (juxt :group_id :perm_type) permissions)
- db-perms (for [[[group-id perm-type] rows] permission-groups
- :when (not (granular? rows))]
- [group-id perm-type (:perm_value (first rows))])
- table-perms (for [[[group-id perm-type] rows] permission-groups
- :when (granular? rows)
- :let [remapped (into {}
- (keep (fn [{:keys [table_id perm_value]}]
- (when-let [new-table-id (-> table_id table-id->name name->new-table-id)]
- [new-table-id perm_value]))
- rows))]
- :when (seq remapped)]
- [group-id perm-type remapped])]
- (doseq [[group-id perm-type perm-value] db-perms]
- (perms/set-database-permission! group-id new-db-id perm-type perm-value))
- (doseq [[group-id perm-type one-table-perms] table-perms]
- (perms/set-table-permissions! group-id perm-type one-table-perms))))
-
-(defn- replace-sample-database!
- "The bundled sample database's engine changed (e.g. H2 -> SQLite on upgrade). The old sample
- Database's tables, fields, and connection details are incompatible with the new engine, so rather
- than remapping content we drop the old sample Database wholesale and extract + sync the new one.
- Deleting the Database cascades to its Cards and those cards' dashcards; Dashboards left empty as a
- result are deleted too.
-
- The cleanup runs unconditionally (an incompatible sample database should never linger), but the new
- Sample Database and its Example collection are only recreated when sample content is enabled - and
- the Example collection is skipped under test endpoints, matching fresh-install seeding.
-
- The Example collection itself is left in place: [[example-content/recreate-example-content!]] reuses it
- (matching by entity id) and replaces only the bundled cards/dashboards, so any content a user filed into
- the Example collection survives the engine swap."
+(defn- table-schema-for-engine
+ "The schema value the sync process assigns to the sample database's tables for a given engine: H2 puts
+ everything in the PUBLIC schema, SQLite reports no schema."
+ [engine]
+ (case engine
+ :h2 "PUBLIC"
+ :sqlite nil))
+
+(defn- migrate-sample-database-engine-in-place!
+ "The only app-db differences between the H2 and
+ SQLite sample databases are the Database record's engine/details and the tables' schema (H2 = \"PUBLIC\",
+ SQLite = nil). So instead of deleting and rebuilding the database, edit those two things in place and
+ re-sync to reconcile the remaining sync-derived field metadata. Every Database/Table/Field id is kept,
+ so sample content, user-created content, permissions, and the Example collection all survive with no
+ remapping, deletion, or re-seeding."
[engine old-sample-db]
- (log/infof "Bundled sample database engine changed from %s to %s; replacing the sample database"
- (:engine old-sample-db) engine)
- ;; Extract the bundled DB file (a multi-MB copy out of the JAR) before opening the transaction, so the file IO
- ;; doesn't hold the app-DB transaction open.
- (let [new-db-details (when (config/load-sample-content?)
- (try-to-extract-sample-database! engine))
- {:keys [new-db permissions-snapshot]}
- (t2/with-transaction [_conn]
- (let [dashboard-ids (sample-database-dashboard-ids (:id old-sample-db))
- snapshot (when new-db-details
- (capture-permissions-snapshot (:id old-sample-db)))]
- (t2/delete! :model/Database (:id old-sample-db))
- (delete-emptied-dashboards! dashboard-ids)
- {:permissions-snapshot snapshot
- :new-db (when new-db-details
- (first (t2/insert-returning-instances! :model/Database
- :name sample-database-name
- :details new-db-details
- :engine engine
- :is_sample true)))}))]
- (when new-db
- (let [synced? (try
- (sync/sync-database! new-db)
- true
- (catch Throwable e
- (log/error e "Failed to sync the replacement sample database")
- false))]
- (when synced?
- ;; The new database's tables exist now, so the old sample DB's access can be restored (replacing
- ;; the defaults granted at creation) before the bundled example content is recreated.
- (when permissions-snapshot
- (apply-permissions-snapshot! (:id new-db) permissions-snapshot))
- (when-not (config/config-bool :mb-enable-test-endpoints)
- (example-content/recreate-example-content! (:id new-db))))))))
+ (log/infof "Migrating sample database engine from %s to %s in place" (:engine old-sample-db) engine)
+ (let [details (try-to-extract-sample-database! engine)]
+ (t2/with-transaction [_conn]
+ (t2/update! :model/Database (:id old-sample-db) {:engine engine, :details details})
+ (t2/update! :model/Table :db_id (:id old-sample-db) {:schema (table-schema-for-engine engine)}))
+ (sync/sync-database! (t2/select-one :model/Database :id (:id old-sample-db)))))
(defn update-sample-database-if-needed!
"Reconcile the existing sample database with the bundled one. When the bundled engine changed
- (H2 <-> SQLite) the old sample database is replaced (see [[replace-sample-database!]]); otherwise
- we just refresh its connection details in case the JAR has moved."
+ (H2 <-> SQLite) the existing sample database is migrated in place to the new engine (see
+ [[migrate-sample-database-engine-in-place!]]); otherwise we just refresh its connection details in case
+ the JAR has moved."
([]
(update-sample-database-if-needed! (t2/select-one :model/Database :is_sample true)))
@@ -231,7 +144,7 @@
(when sample-db
(let [engine (sample-database-engine)]
(if (not= (:engine sample-db) engine)
- (replace-sample-database! engine sample-db)
+ (migrate-sample-database-engine-in-place! engine sample-db)
(let [intended (try-to-extract-sample-database! engine)]
(when (not= (:details sample-db) intended)
(t2/update! :model/Database (:id sample-db) {:details intended}))))))))
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 1bf7dabd0c4..f587b8a790c 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -477,58 +477,53 @@
(t2/define-before-update :model/Database
[database]
+ ;; Note: the "sample database may not be edited" policy is enforced at the API layer
+ ;; ([[metabase.warehouses-rest.api]] PUT /:id), so internally-derived updates - e.g. the sample
+ ;; database engine migration in [[metabase.sample-data.impl]] - can change the engine here.
(let [changes (t2/changes database)
{new-engine :engine
new-settings :settings} changes
- {is-sample? :is_sample
- existing-settings :settings
+ {existing-settings :settings
existing-engine :engine} (t2/original database)
new-engine (some-> new-engine keyword)]
- (if (and is-sample?
- new-engine
- (not= new-engine existing-engine))
- (throw (ex-info (trs "The engine on a sample database cannot be changed.")
- {:status-code 400
- :existing-engine existing-engine
- :new-engine new-engine}))
- (u/prog1 (cond-> database
- ;; If the engine doesn't support nested field columns, `json_unfolding` must be nil
- (and (some? (:details changes))
- (not (driver.u/supports? (or new-engine existing-engine) :nested-field-columns database)))
- (update :details dissoc :json_unfolding)
-
- (or
- ;if there is any changes in user control setting
- (some? (get-in changes [:details :let-user-control-scheduling]))
- ;; if the let user control scheduling is already on, we should always try to re-infer it
- (get-in database [:details :let-user-control-scheduling])
- ;; if there is a changes in schedules, make sure it respects the settings
- (some some? [(:cache_field_values_schedule changes) (:metadata_sync_schedule changes)]))
- infer-db-schedules
-
- (or (some? (:details changes))
- (some? (:write_data_details changes))
- (some? (:admin_details changes)))
- secret/handle-incoming-client-secrets!
-
- (:uploads_enabled changes)
- maybe-disable-uploads-for-all-dbs!)
- ;; This maintains a constraint that if a driver doesn't support actions, it can never be enabled
- ;; If we drop support for actions for a driver, we'd need to add a migration to disable actions for all databases
- (when (and (:database-enable-actions (or new-settings existing-settings))
- (not (driver.u/supports? (or new-engine existing-engine) :actions database)))
- (throw (ex-info (trs "The database does not support actions.")
- {:status-code 400
- :existing-engine existing-engine
- :new-engine new-engine})))
- ;; This maintains a constraint that if a driver doesn't support data editing, it can never be enabled
- ;; If we drop support for a driver, we'd need to add a migration to disable it for all databases
- (when (and (:database-enable-table-editing (or new-settings existing-settings))
- (not (driver.u/supports? (or new-engine existing-engine) :actions/data-editing database)))
- (throw (ex-info (trs "The database does not support table editing.")
- {:status-code 400
- :existing-engine existing-engine
- :new-engine new-engine})))))))
+ (u/prog1 (cond-> database
+ ;; If the engine doesn't support nested field columns, `json_unfolding` must be nil
+ (and (some? (:details changes))
+ (not (driver.u/supports? (or new-engine existing-engine) :nested-field-columns database)))
+ (update :details dissoc :json_unfolding)
+
+ (or
+ ;if there is any changes in user control setting
+ (some? (get-in changes [:details :let-user-control-scheduling]))
+ ;; if the let user control scheduling is already on, we should always try to re-infer it
+ (get-in database [:details :let-user-control-scheduling])
+ ;; if there is a changes in schedules, make sure it respects the settings
+ (some some? [(:cache_field_values_schedule changes) (:metadata_sync_schedule changes)]))
+ infer-db-schedules
+
+ (or (some? (:details changes))
+ (some? (:write_data_details changes))
+ (some? (:admin_details changes)))
+ secret/handle-incoming-client-secrets!
+
+ (:uploads_enabled changes)
+ maybe-disable-uploads-for-all-dbs!)
+ ;; This maintains a constraint that if a driver doesn't support actions, it can never be enabled
+ ;; If we drop support for actions for a driver, we'd need to add a migration to disable actions for all databases
+ (when (and (:database-enable-actions (or new-settings existing-settings))
+ (not (driver.u/supports? (or new-engine existing-engine) :actions database)))
+ (throw (ex-info (trs "The database does not support actions.")
+ {:status-code 400
+ :existing-engine existing-engine
+ :new-engine new-engine})))
+ ;; This maintains a constraint that if a driver doesn't support data editing, it can never be enabled
+ ;; If we drop support for a driver, we'd need to add a migration to disable it for all databases
+ (when (and (:database-enable-table-editing (or new-settings existing-settings))
+ (not (driver.u/supports? (or new-engine existing-engine) :actions/data-editing database)))
+ (throw (ex-info (trs "The database does not support table editing.")
+ {:status-code 400
+ :existing-engine existing-engine
+ :new-engine new-engine}))))))
(t2/define-after-update :model/Database
[database]
diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.clj
index 33d7130f95f..56062d4608a 100644
--- a/test/metabase/app_db/custom_migrations_test.clj
+++ b/test/metabase/app_db/custom_migrations_test.clj
@@ -2938,200 +2938,3 @@
(is (= "metabase-transform" (:data_source provisional)))
(is (= "computed" (:data_authority provisional)))
(is (= "New Target Table" (:display_name provisional)))))))))
-
-(deftest ^:mb/old-migrations-test migrate-away-from-sqlite-sample-database-on-downgrade-rollback-test
- (testing "Downgrading past the changeset (the real rollback path - the target version has none of the new
- sample database code) removes the SQLite sample database and the whole Example collection tree that
- the CreateSampleContent migration installed"
- ;; Start early enough that CreateSampleContent runs under our binding and installs the real SQLite Sample
- ;; Database and Example collection tree - the exact state a SQLite-sample version leaves to be torn down.
- (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]
- (binding [custom-migrations/*create-sample-content* true]
- (migrate!))
- (let [sample-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]
- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))
- sample-coll-ids (mapv :id (t2/query {:select [:id] :from [:collection] :where [:= :is_sample true]}))]
- (testing "precondition: the SQLite sample database and its Example collections exist after upgrade"
- (is (some? sample-db-id))
- (is (seq sample-coll-ids)))
- (migrate! :down 62)
- (testing "the SQLite sample database is gone"
- (is (not (t2/exists? (t2/table-name :model/Database) :id sample-db-id))))
- (testing "the entire outgoing Example collection tree is deleted - the cascade is complete"
- (is (zero? (t2/count (t2/table-name :model/Collection) :id [:in sample-coll-ids]))))))))
-
-(deftest migrate-away-from-sqlite-sample-database-on-downgrade-test
- (testing "Downgrade removes the SQLite sample database and the content it leaves empty, keeping everything else"
- (mt/with-temp
- [:model/Database sample {:engine :sqlite, :is_sample true, :details {:db "mem:sample"}}
- :model/Database other {:engine :h2, :details {:db "mem:other"}}
- :model/Card other-card {:database_id (:id other)}
- :model/Dashboard sample-dash {}
- :model/Dashboard mixed-dash {}
- :model/Collection examples {:name "Examples", :is_sample true}
- :model/Collection ecommerce {:name "E-commerce", :is_sample true, :location (str "/" (:id examples) "/")}
- :model/Collection keep-coll {:name "Keep me"}]
- ;; Mirror the bundled sample content's fan-out: 8 tables x 7 fields, 39 cards, plus dashcards,
- ;; series, and tabs. MySQL 9.7 resolves a multi-level ON DELETE CASCADE of this shape
- ;; incompletely (it deletes the tables but orphans most cards and all fields), so the cleanup
- ;; must not rely on DB-level cascade. This fixture makes that regression visible.
- (let [table-ids (vec (t2/insert-returning-pks! :model/Table
- (for [i (range 8)]
- {:db_id (:id sample)
- :name (str "sample_table_" i)
- :active true})))
- _ (t2/insert! :model/Field
- (for [tid table-ids
- i (range 7)]
- {:table_id tid
- :name (str "field_" i)
- :base_type :type/Text
- :database_type "TEXT"
- :position i}))
- card-ids (vec (t2/insert-returning-pks! :model/Card
- (for [i (range 39)]
- {:name (str "sample card " i)
- :display "table"
- :dataset_query {}
- :visualization_settings {}
- :creator_id (mt/user->id :rasta)
- :database_id (:id sample)
- :table_id (nth table-ids (mod i (count table-ids)))})))
- [dc1 _dc2 dc3] (t2/insert-returning-pks! :model/DashboardCard
- [{:dashboard_id (:id sample-dash) :card_id (first card-ids)
- :size_x 4 :size_y 4 :row 0 :col 0}
- {:dashboard_id (:id mixed-dash) :card_id (second card-ids)
- :size_x 4 :size_y 4 :row 0 :col 0}
- {:dashboard_id (:id mixed-dash) :card_id (:id other-card)
- :size_x 4 :size_y 4 :row 0 :col 4}])
- _ (t2/insert! :model/DashboardCardSeries {:dashboardcard_id dc1
- :card_id (nth card-ids 2)
- :position 0})
- _ (t2/insert! :model/DashboardTab {:dashboard_id (:id sample-dash)
- :name "Tab 1"
- :position 0})
- ;; parameter_card rows orphan in three ways; a fourth must survive.
- [pc-src
- pc-card-owner
- pc-dash-owner
- pc-keep] (t2/insert-returning-pks!
- :model/ParameterCard
- [;; value source is a sample card -> delete
- {:card_id (first card-ids) :parameterized_object_type "dashboard"
- :parameterized_object_id (:id mixed-dash) :parameter_id "src"}
- ;; parameterized object is a sample card -> delete
- {:card_id (:id other-card) :parameterized_object_type "card"
- :parameterized_object_id (second card-ids) :parameter_id "card-owner"}
- ;; parameterized object is a deleted sample dashboard -> delete
- {:card_id (:id other-card) :parameterized_object_type "dashboard"
- :parameterized_object_id (:id sample-dash) :parameter_id "dash-owner"}
- ;; non-sample source on a surviving dashboard -> keep
- {:card_id (:id other-card) :parameterized_object_type "dashboard"
- :parameterized_object_id (:id mixed-dash) :parameter_id "keep"}])]
- ;; Collection creation auto-grants an "All Users" permission row (object "/collection/<id>/") for each
- ;; sample collection; the migration must clean those up too.
- (#'custom-migrations/remove-sqlite-sample-database-on-downgrade!)
- (testing "the SQLite sample database and all of its child content are deleted, with no orphans"
- (is (not (t2/exists? :model/Database :id (:id sample))))
- (is (zero? (t2/count :metabase_table :db_id (:id sample))))
- (is (zero? (t2/count :metabase_field :table_id [:in table-ids])))
- (is (zero? (t2/count :report_card :database_id (:id sample))))
- (is (zero? (t2/count :report_dashboardcard :card_id [:in card-ids])))
- (is (zero? (t2/count :dashboardcard_series :card_id [:in card-ids]))))
- (testing "a dashboard left empty by the deletion is deleted, along with its tabs"
- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))
- (is (zero? (t2/count :dashboard_tab :dashboard_id (:id sample-dash)))))
- (testing "parameter_card rows referencing a deleted sample card or dashboard are removed"
- (is (not (t2/exists? :model/ParameterCard :id pc-src)))
- (is (not (t2/exists? :model/ParameterCard :id pc-card-owner)))
- (is (not (t2/exists? :model/ParameterCard :id pc-dash-owner)))
- (testing "but an unrelated parameter_card is kept"
- (is (t2/exists? :model/ParameterCard :id pc-keep))))
- (testing "the sample Example collections and their permission records are deleted"
- (is (not (t2/exists? :model/Collection :id (:id examples))))
- (is (not (t2/exists? :model/Collection :id (:id ecommerce))))
- (is (zero? (t2/count :permissions :object [:in [(format "/collection/%d/" (:id examples))
- (format "/collection/%d/read/" (:id examples))]]))))
- (testing "a dashboard that still has other cards, and unrelated content, is kept"
- (is (t2/exists? :model/Dashboard :id (:id mixed-dash)))
- (is (t2/exists? :report_dashboardcard :id dc3))
- (is (t2/exists? :model/Card :id (:id other-card)))
- (is (t2/exists? :model/Database :id (:id other)))
- (is (t2/exists? :model/Collection :id (:id keep-coll))))))))
-
-(deftest ^:mb/old-migrations-test downgrade-keeps-user-content-and-deletes-sample-dependents-test
- (testing "Downgrade deletes everything that depends on the sample DB (transitively, even user-created), then keeps
- any Example collection a user still has their own content in - leaving the RESTRICT FKs intact rather than
- aborting on them"
- (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]
- (binding [custom-migrations/*create-sample-content* true]
- (migrate!))
- (let [examples-id (:id (t2/query-one {:select [:id] :from [:collection]
- :where [:= :is_sample true] :order-by [[:id :asc]]}))
- ecommerce-id (:id (t2/query-one {:select [:id] :from [:collection]
- :where [:= :is_sample true] :order-by [[:id :desc]]}))
- sqlite-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]
- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))
- sample-card (:id (t2/query-one {:select [:id] :from [:report_card] :where [:= :database_id sqlite-db-id] :limit 1}))
- ins (fn [t r] (first (t2/insert-returning-pks! t r)))
- other-db (ins :metabase_database {:name "User DB" :engine "h2" :is_sample false
- :details "{}" :created_at :%now :updated_at :%now})
- ;; a user question that doesn't touch the sample DB, filed in Examples -> must survive
- user-card (ins :report_card {:name "keep me" :display "table" :dataset_query "{}"
- :visualization_settings "{}" :creator_id 13371338 :database_id other-db
- :collection_id examples-id :created_at :%now :updated_at :%now})
- ;; a user model built on a sample card -> depends transitively -> must be deleted
- user-model (ins :report_card {:name "built on sample" :display "table" :dataset_query "{}"
- :visualization_settings "{}" :creator_id 13371338 :database_id other-db
- :source_card_id sample-card :collection_id examples-id
- :created_at :%now :updated_at :%now})
- ;; a curated table + a transform filed in E-commerce (RESTRICT FKs) -> keep that collection, FKs intact
- table-id (ins :metabase_table {:name "curated" :active true :db_id other-db
- :collection_id ecommerce-id :created_at :%now :updated_at :%now})
- transform-id (ins :transform {:name "tf" :source "{}" :target "{}" :source_type "query"
- :collection_id ecommerce-id :created_at :%now :updated_at :%now})
- exists? (fn [t id] (boolean (seq (t2/query {:select [1] :from [t] :where [:= :id id] :limit 1}))))
- coll-of (fn [t id] (:collection_id (t2/query-one {:select [:collection_id] :from [t] :where [:= :id id]})))]
- (migrate! :down 62)
- (testing "the sample database and everything depending on it - including the user model - are deleted"
- (is (not (exists? :metabase_database sqlite-db-id)))
- (is (not (exists? :report_card sample-card)))
- (is (not (exists? :report_card user-model))))
- (testing "a user question that does not depend on the sample DB survives"
- (is (exists? :report_card user-card)))
- (testing "Example collections holding surviving user content are kept"
- (is (exists? :collection examples-id))
- (is (exists? :collection ecommerce-id)))
- (testing "the RESTRICT-FK table and transform survive with their collection_id intact (no abort, no null-out)"
- (is (exists? :metabase_table table-id))
- (is (= ecommerce-id (coll-of :metabase_table table-id)))
- (is (exists? :transform transform-id))
- (is (= ecommerce-id (coll-of :transform transform-id))))))))
-
-(deftest ^:mb/old-migrations-test downgrade-leaves-no-orphaned-card-children-test
- ;; B1 characterization: the migration hand-deletes only the dashcard tables and lets DB-level ON DELETE CASCADE
- ;; clear report_card's other ~11 children (query_field, query_table, card_bookmark, report_cardfavorite, ...).
- ;; Its own comment warns that MySQL 9.7 resolves cascades incompletely. This test seeds those children for a
- ;; sample card and asserts none survive the card's deletion - i.e. it FAILS if the relied-on cascade leaves
- ;; orphans. It deliberately asserts only on the outcome (no orphans), not on how the cleanup is done.
- (testing "downgrade leaves no child rows pointing at a deleted sample card"
- (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]
- (binding [custom-migrations/*create-sample-content* true]
- (migrate!))
- (let [internal-user 13371338
- sqlite-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]
- :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))
- sample-card (:id (t2/query-one {:select [:id] :from [:report_card] :where [:= :database_id sqlite-db-id] :limit 1}))
- ins (fn [t r] (first (t2/insert-returning-pks! t r)))]
- ;; children of report_card that the migration leaves to ON DELETE CASCADE (not hand-deleted)
- (ins :query_field {:card_id sample-card :column "TOTAL" :explicit_reference true})
- (ins :query_table {:card_id sample-card :table "ORDERS"})
- (ins :card_bookmark {:user_id internal-user :card_id sample-card})
- (ins :report_cardfavorite {:card_id sample-card :owner_id internal-user :created_at :%now :updated_at :%now})
- (#'custom-migrations/remove-sqlite-sample-database-on-downgrade!)
- (testing "the sample card was deleted"
- (is (not (t2/exists? :report_card :id sample-card))))
- (testing "no child table is left with a row pointing at the deleted card"
- (doseq [table [:query_field :query_table :card_bookmark :report_cardfavorite]]
- (is (zero? (t2/count table :card_id sample-card))
- (str table " has an orphaned row referencing the deleted sample card"))))))))
diff --git a/test/metabase/sample_data/example_content_test.clj b/test/metabase/sample_data/example_content_test.clj
deleted file mode 100644
index d8bd0060301..00000000000
--- a/test/metabase/sample_data/example_content_test.clj
+++ /dev/null
@@ -1,141 +0,0 @@
-(ns metabase.sample-data.example-content-test
- (:require
- [clojure.test :refer :all]
- [clojure.walk :as walk]
- [metabase.query-processor :as qp]
- [metabase.sample-data.example-content :as example-content]
- [metabase.sample-data.impl]
- [metabase.sync.core :as sync]
- [metabase.test :as mt]
- [metabase.util.json :as json]
- [toucan2.core :as t2]))
-
-(deftest remap-blob-parameters-preserves-order-test
- (testing "remap-blob keeps a card/dashboard's parameters in their designed order rather than re-sorting by :id"
- (let [blob (json/encode [{:id "zzzz" :name "Date Range" :type "string/="}
- {:id "aaaa" :name "Vendor" :type "string/="}])
- out (json/decode+kw (#'example-content/remap-blob {} :parameters blob))]
- (is (= ["Date Range" "Vendor"] (mapv :name out))
- "parameters must stay in original order, not be alphabetized by :id"))))
-
-(deftest remap-collection-location-test
- (let [remap #'example-content/remap-collection-location]
- (testing "root location is unchanged"
- (is (= "/" (remap {2 20} "/"))))
- (testing "each path segment is remapped independently"
- (is (= "/20/" (remap {2 20} "/2/")))
- (is (= "/20/30/" (remap {2 20, 3 30} "/2/3/"))))
- (testing "a new id colliding with a still-pending old id does not double-substitute"
- ;; old reduce-of-str/replace would turn \"/5/12/\" into \"/99/99/\"; per-segment remapping keeps them distinct
- (is (= "/12/99/" (remap {5 12, 12 99} "/5/12/"))))
- (testing "segments without a mapping keep their original id"
- (is (= "/2/7/" (remap {2 2} "/2/7/"))))))
-
-(defn- sample-database-db []
- {:details (#'metabase.sample-data.impl/try-to-extract-sample-database! :sqlite)
- :engine :sqlite
- :name "Sample Database"})
-
-(defn- field-ids-in-query
- "All numeric field ids referenced anywhere in a (pMBQL) dataset_query."
- [query]
- (let [ids (volatile! #{})]
- (walk/postwalk
- (fn [x]
- (when (and (vector? x) (= :field (first x)))
- (let [id (last x)]
- (when (number? id) (vswap! ids conj id))))
- x)
- query)
- @ids))
-
-(defn- table-ids-in-query
- "All numeric :source-table ids referenced anywhere in a (pMBQL) dataset_query."
- [query]
- (let [ids (volatile! #{})]
- (walk/postwalk
- (fn [x]
- (when (and (map? x) (number? (:source-table x)))
- (vswap! ids conj (:source-table x)))
- x)
- query)
- @ids))
-
-(deftest example-card-queries-run-after-id-bumping-upgrade-test
- (testing "After an upgrade that bumped app-db ids past the EDN-hardcoded range, every recreated example card runs.
- Cards that build on another example card carry a `\"card__N\"` :source-table; export-mbql only remaps a
- *numeric* :source-table, so these stay pinned to the EDN card id and now point at an unrelated/missing card."
- (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard
- :model/Dimension :model/Permissions]
- ;; Bump table/field ids past the EDN range (8 tables, 71 fields).
- (mt/with-temp [:model/Database other {}
- :model/Table ot {:db_id (:id other)}]
- (doseq [i (range 200)]
- (t2/insert! :model/Field {:table_id (:id ot) :name (str "filler_" i) :base_type :type/Integer
- :database_type "INT" :position i}))
- ;; Bump card ids past the EDN range (39 cards) so the EDN card ids (e.g. card__1) are taken by these
- ;; unrelated filler cards rather than by the recreated example cards.
- (dotimes [_ 60]
- (t2/insert! :model/Card (mt/with-temp-defaults :model/Card)))
- (mt/with-temp [:model/Database db (sample-database-db)]
- (sync/sync-database! db)
- (example-content/recreate-example-content! (:id db))
- (let [cards (->> (t2/select :model/Card :database_id (:id db))
- (filter (comp #{:question :model :metric} :type))
- (sort-by :id))]
- (is (seq cards) "example cards were recreated")
- (doseq [card cards]
- (testing (format "%s %s (card %s)" (:type card) (pr-str (:name card)) (:id card))
- (let [result (try (qp/process-query (:dataset_query card))
- (catch Throwable e {:error (ex-message e)}))]
- (is (= nil (:error result))
- (format "query failed: %s" (:error result))))))))))))
-
-(deftest recreate-example-content-test
- (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard
- :model/Dimension :model/Permissions]
- ;; Pre-create an unrelated database with tables/fields so the sample DB's synced table/field ids do
- ;; NOT coincide with the EDN ids - otherwise the remap would be an identity no-op and prove nothing.
- (mt/with-temp [:model/Database other {}
- :model/Table ot {:db_id (:id other)}]
- (doseq [i (range 120)]
- (t2/insert! :model/Field {:table_id (:id ot) :name (str "filler_" i) :base_type :type/Integer
- :database_type "INT" :position i}))
- (mt/with-temp [:model/Database db (sample-database-db)]
- (sync/sync-database! db)
- (example-content/recreate-example-content! (:id db))
- (let [cards (t2/select :model/Card :database_id (:id db))
- coll-ids (set (keep :collection_id cards))
- valid-fid (set (t2/select-fn-set :id :model/Field
- {:join [[:metabase_table :t] [:= :t.id :metabase_field.table_id]]
- :where [:= :t.db_id (:id db)]}))
- valid-tid (set (t2/select-fn-set :id :model/Table :db_id (:id db)))]
- (testing "the example cards are recreated against the new sample database"
- (is (= 39 (count cards)))
- (is (every? #(= (:id db) (:database_id %)) cards)))
- (testing "every card lives in the example collection"
- (is (seq coll-ids))
- (is (every? :is_sample (t2/select :model/Collection :id [:in coll-ids]))))
- (testing "every field/table reference in every card resolves to a real column of the new database"
- (doseq [card cards]
- (let [q (:dataset_query card)]
- (doseq [fid (field-ids-in-query q)]
- (is (contains? valid-fid fid)
- (format "card %s references missing field %s" (:name card) fid)))
- (doseq [st (table-ids-in-query q)]
- (is (contains? valid-tid st)
- (format "card %s references missing table %s" (:name card) st))))))
- (testing "the dashboard and its dashcards are recreated and resolve to recreated cards"
- (let [dash (t2/select-one :model/Dashboard :collection_id [:in coll-ids])
- dashcards (t2/select :model/DashboardCard :dashboard_id (:id dash))
- card-ids (set (map :id cards))]
- (is (some? dash))
- (is (pos? (count dashcards)))
- (is (every? #(or (nil? (:card_id %)) (contains? card-ids (:card_id %))) dashcards))))
- (testing "a recreated card actually runs against the new sample database"
- (let [runnable (first (filter #(and (nil? (:source_card_id %))
- (= :question (:type %))
- (seq (table-ids-in-query (:dataset_query %))))
- cards))]
- (is (some? runnable))
- (is (pos? (count (mt/rows (qp/process-query (:dataset_query runnable)))))))))))))
diff --git a/test/metabase/sample_data/impl_test.clj b/test/metabase/sample_data/impl_test.clj
index e2fdae0429c..e5a0924c2dc 100644
--- a/test/metabase/sample_data/impl_test.clj
+++ b/test/metabase/sample_data/impl_test.clj
@@ -9,13 +9,12 @@
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.permissions.core :as perms]
[metabase.plugins.impl :as plugins]
- [metabase.sample-data.example-content :as example-content]
+ [metabase.query-processor :as qp]
[metabase.sample-data.impl :as sample-data]
[metabase.sync.core :as sync]
[metabase.sync.task.sync-databases-test :as task.sync-databases-test]
[metabase.test :as mt]
[metabase.util :as u]
- [metabase.util.json :as json]
[metabase.warehouse-schema.models.field-values :as field-values]
[metabase.warehouses-rest.api-test :as api.database-test]
[toucan2.core :as t2])
@@ -54,6 +53,79 @@
;;; ----------------------------------------------------- Tests ------------------------------------------------------
+(deftest migrate-sample-database-engine-in-place-test
+ (testing "Upgrade path: migrating the sample DB from H2 to SQLite in place keeps every Database/Table/Field
+ id, so content referencing those ids survives with no remapping and still queries correctly."
+ (mt/with-model-cleanup [:model/Database :model/Card]
+ ;; Install the pre-upgrade (v62-shape) H2 sample database and a user question that references a field.
+ (let [h2-db (t2/insert-returning-instance! :model/Database
+ {:name "Sample Database" :engine :h2 :is_sample true
+ :details (#'sample-data/try-to-extract-sample-database! :h2)})]
+ (sync/sync-database! h2-db)
+ (let [orders-id (t2/select-one-pk :model/Table :db_id (:id h2-db) :name "ORDERS")
+ total-id (t2/select-one-pk :model/Field :table_id orders-id :name "TOTAL")
+ user-card (t2/insert-returning-instance! :model/Card
+ {:name "user q" :database_id (:id h2-db) :table_id orders-id
+ :display "scalar" :visualization_settings {} :creator_id (mt/user->id :rasta)
+ :dataset_query {:database (:id h2-db) :type :query
+ :query {:source-table orders-id
+ :aggregation [[:sum [:field total-id nil]]]}}})
+ before-tables (t2/select-fn-set :id :model/Table :db_id (:id h2-db))
+ before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables])]
+ (is (= #{"PUBLIC"} (t2/select-fn-set :schema :model/Table :db_id (:id h2-db)))
+ "precondition: H2 tables live in the PUBLIC schema")
+ ;; ---- the migration under test ----
+ (#'sample-data/migrate-sample-database-engine-in-place! :sqlite (t2/select-one :model/Database :id (:id h2-db)))
+ (testing "the database record is now SQLite"
+ (is (= :sqlite (:engine (t2/select-one :model/Database :id (:id h2-db))))))
+ (testing "the same tables remain (no new ones), now with schema = nil"
+ (is (= before-tables (t2/select-fn-set :id :model/Table :db_id (:id h2-db))))
+ (is (= #{nil} (t2/select-fn-set :schema :model/Table :db_id (:id h2-db)))))
+ (testing "the same fields remain (ids preserved), so embedded query refs stay valid"
+ (is (= before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables]))))
+ (testing "the user card survives with its id and still queries the (now SQLite) sample DB"
+ (is (t2/exists? :model/Card :id (:id user-card)))
+ (let [result (qp/process-query (:dataset_query (t2/select-one :model/Card :id (:id user-card))))]
+ (is (= :completed (:status result)))
+ (is (pos? (count (mt/rows result)))))))))))
+
+(deftest migrate-sample-database-engine-in-place-downgrade-test
+ (testing "Downgrade path: migrating the sample DB from SQLite back to H2 in place keeps every id, so
+ sample and user content survive with no remapping and still query correctly."
+ (mt/with-model-cleanup [:model/Database :model/Collection :model/Card :model/Dashboard]
+ ;; Install the SQLite sample database (the state a newer version leaves behind) and a user question
+ ;; that references a field.
+ (let [sqlite-db (t2/insert-returning-instance! :model/Database
+ {:name "Sample Database" :engine :sqlite :is_sample true
+ :details (#'sample-data/try-to-extract-sample-database! :sqlite)})]
+ (sync/sync-database! sqlite-db)
+ (let [orders-id (t2/select-one-pk :model/Table :db_id (:id sqlite-db) :name "ORDERS")
+ total-id (t2/select-one-pk :model/Field :table_id orders-id :name "TOTAL")
+ user-card (t2/insert-returning-instance! :model/Card
+ {:name "user q" :database_id (:id sqlite-db) :table_id orders-id
+ :display "scalar" :visualization_settings {} :creator_id (mt/user->id :rasta)
+ :dataset_query {:database (:id sqlite-db) :type :query
+ :query {:source-table orders-id
+ :aggregation [[:sum [:field total-id nil]]]}}})
+ before-tables (t2/select-fn-set :id :model/Table :db_id (:id sqlite-db))
+ before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables])]
+ (is (= #{nil} (t2/select-fn-set :schema :model/Table :db_id (:id sqlite-db)))
+ "precondition: SQLite tables have a nil schema")
+ ;; ---- the migration under test ----
+ (#'sample-data/migrate-sample-database-engine-in-place! :h2 (t2/select-one :model/Database :id (:id sqlite-db)))
+ (testing "the database record is now H2"
+ (is (= :h2 (:engine (t2/select-one :model/Database :id (:id sqlite-db))))))
+ (testing "the same tables remain (no new ones), now with schema = PUBLIC"
+ (is (= before-tables (t2/select-fn-set :id :model/Table :db_id (:id sqlite-db))))
+ (is (= #{"PUBLIC"} (t2/select-fn-set :schema :model/Table :db_id (:id sqlite-db)))))
+ (testing "the same fields remain (ids preserved)"
+ (is (= before-fields (t2/select-fn-set :id :model/Field :table_id [:in before-tables]))))
+ (testing "the user card survives with its id and still queries the (now H2) sample DB"
+ (is (t2/exists? :model/Card :id (:id user-card)))
+ (let [result (qp/process-query (:dataset_query (t2/select-one :model/Card :id (:id user-card))))]
+ (is (= :completed (:status result)))
+ (is (pos? (count (mt/rows result)))))))))))
+
(def ^:private extracted-db-path-regex #".*plugins/sample-database\.sqlite$")
(deftest extract-sample-database-test
@@ -132,142 +204,6 @@
(is (re-matches extracted-db-path-regex
(get-in (t2/select-one :model/Database :id (:id db)) [:details :db]))))))))
-(deftest replace-sample-database-on-engine-change-test
- (testing "When the bundled engine changed (H2 -> SQLite), the old sample DB and its dependent content are removed"
- (mt/with-temp
- [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}
- :model/Database other-db {:engine :h2, :details {:db "mem:other"}}
- :model/Card sample-card {:database_id (:id old-sample)}
- :model/Card other-card {:database_id (:id other-db)}
- :model/Dashboard sample-dash {}
- :model/Dashboard mixed-dash {}
- :model/DashboardCard _ {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}
- :model/DashboardCard _ {:dashboard_id (:id mixed-dash), :card_id (:id sample-card)}
- :model/DashboardCard _ {:dashboard_id (:id mixed-dash), :card_id (:id other-card)}
- :model/Collection examples {:name "Examples", :is_sample true}
- :model/Collection ecommerce {:name "E-commerce", :is_sample true, :location (str "/" (:id examples) "/")}
- :model/Collection keep-coll {:name "Keep me"}]
- (let [bundled-engine (#'sample-data/sample-database-engine)
- synced-db-ids (atom [])]
- ;; recreate (collection reuse) is exercised separately; here we only check the database swap + cleanup.
- (with-redefs [example-content/recreate-example-content! (constantly nil)]
- (mt/with-dynamic-fn-redefs [sync/sync-database! (fn [db] (swap! synced-db-ids conj (:id db)) db)]
- (#'sample-data/update-sample-database-if-needed! old-sample)))
- (testing "the old sample DB is deleted, cascading to its cards"
- (is (not (t2/exists? :model/Database :id (:id old-sample))))
- (is (not (t2/exists? :model/Card :id (:id sample-card)))))
- (testing "a dashboard emptied by the deletion is deleted"
- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash)))))
- (testing "a dashboard that still has other cards is kept"
- (is (t2/exists? :model/Dashboard :id (:id mixed-dash)))
- (is (t2/exists? :model/Card :id (:id other-card))))
- (testing "the Example collections are preserved (reused on recreate), not deleted by the engine swap"
- (is (t2/exists? :model/Collection :id (:id examples)))
- (is (t2/exists? :model/Collection :id (:id ecommerce)))
- (is (t2/exists? :model/Collection :id (:id keep-coll))))
- (testing "a new sample database with the bundled engine is created and synced"
- ;; replace-sample-database! inserts the new sample DB outside any with-temp, so key off the synced id
- ;; (authoritative) rather than (select-one :is_sample true), which would see leaks from other tests.
- (is (= 1 (count @synced-db-ids)) "the new database (not the old one) is what gets synced")
- (let [new-db (t2/select-one :model/Database :id (first @synced-db-ids))]
- (is (some? new-db))
- (is (not= (:id old-sample) (:id new-db)))
- (is (= bundled-engine (:engine new-db)))
- ;; clean it up so the leaked is_sample row can't pollute tests that select on is_sample.
- (t2/delete! :model/Database :id (:id new-db))))))))
-
-(deftest replace-sample-database-cascades-to-synced-schema-test
- (testing "On H2 -> SQLite replacement, deleting the old sample DB also removes its synced schema (Tables, Fields,
- Dimensions) and dashboard tabs - i.e. every entity belonging to the old H2 sample database is gone"
- (mt/with-temp
- [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}
- :model/Table table {:db_id (:id old-sample)}
- :model/Field field {:table_id (:id table)}
- :model/Dimension dimension {:field_id (:id field)}
- :model/Card sample-card {:database_id (:id old-sample)}
- :model/Dashboard sample-dash {}
- :model/DashboardTab tab {:dashboard_id (:id sample-dash)}
- :model/DashboardCard _ {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}
- :model/Collection examples {:name "Examples", :is_sample true}]
- ;; Skip recreating the replacement Sample Database (load-sample-content? false): this test only checks the
- ;; old schema cascades away, and recreating it would leak an is_sample row other tests select on.
- (with-redefs [config/load-sample-content? (constantly false)]
- (#'sample-data/update-sample-database-if-needed! old-sample))
- (testing "the old sample DB is deleted"
- (is (not (t2/exists? :model/Database :id (:id old-sample)))))
- (testing "its synced schema cascades away"
- (is (not (t2/exists? :model/Table :id (:id table))))
- (is (not (t2/exists? :model/Field :id (:id field))))
- (is (not (t2/exists? :model/Dimension :id (:id dimension)))))
- (testing "its card and the dashboard emptied by the card's removal are deleted, along with the dashboard's tabs"
- (is (not (t2/exists? :model/Card :id (:id sample-card))))
- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))
- (is (not (t2/exists? :model/DashboardTab :id (:id tab)))))
- (testing "the Example collection is preserved (the engine swap no longer deletes it)"
- (is (t2/exists? :model/Collection :id (:id examples)))))))
-
-(deftest replace-sample-database-survives-dangling-visualizer-ref-test
- (testing "The engine swap completes when a sample dashcard has a dangling visualizer ref"
- (mt/with-temp
- [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}
- :model/Card sample-card {:database_id (:id old-sample)}
- :model/Dashboard sample-dash {}
- :model/DashboardCard dc {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}]
- ;; Inject a dangling visualizer ref via raw SQL so the model's hooks don't reject/rewrite it.
- (t2/query-one {:update (t2/table-name :model/DashboardCard)
- :set {:visualization_settings
- (json/encode {:visualization
- {:columnValuesMapping
- {:COLUMN_1 [{:sourceId "card:gEnfWx10SmfjiccZpcGrj"}]}}})}
- :where [:= :id (:id dc)]})
- (with-redefs [config/load-sample-content? (constantly false)]
- (testing "the replacement completes instead of throwing on the dangling ref"
- (is (nil? (#'sample-data/update-sample-database-if-needed! old-sample)))))
- (testing "the old sample DB and the dashboard emptied by its removal are deleted"
- (is (not (t2/exists? :model/Database :id (:id old-sample))))
- (is (not (t2/exists? :model/Card :id (:id sample-card))))
- (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))))))
-
-(def ^:private edn-example-collection-entity-id
- "Stable entity id of the bundled Examples collection in sample-content.edn."
- "HyB3nRtqb7pBPhFG26evI")
-
-(deftest recreate-example-content-reuses-collection-test
- (testing "recreate-example-content! reuses an existing Example collection (matched by entity id) and preserves the
- user content filed into it, instead of deleting it and creating a brand new collection"
- (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard
- :model/Dimension :model/Permissions]
- (with-temp-sample-database-db [db]
- (mt/with-temp
- [:model/Collection examples {:name "Examples", :is_sample true, :location "/"
- :entity_id edn-example-collection-entity-id}
- ;; a question a user filed into the Example collection - must survive the engine swap
- :model/Card user-card {:name "user question", :collection_id (:id examples), :database_id (:id db)}]
- (example-content/recreate-example-content! (:id db))
- (testing "the existing Example collection is reused, not duplicated"
- (is (t2/exists? :model/Collection :id (:id examples)))
- (is (= 1 (t2/count :model/Collection :entity_id edn-example-collection-entity-id))))
- (testing "the user's content survives, still filed in the reused Example collection"
- (is (t2/exists? :model/Card :id (:id user-card)))
- (is (= (:id examples) (:collection_id (t2/select-one :model/Card :id (:id user-card))))))
- (testing "the bundled sample content is recreated on the sample database, in the reused collection"
- (is (pos? (t2/count :model/Card :database_id (:id db))))
- (is (t2/exists? :model/Dashboard :collection_id (:id examples)))))))))
-
-(deftest replace-sample-database-skips-content-when-sync-fails-test
- (testing "When the replacement sync fails, example content is NOT recreated - recreating against an unsynced DB
- remaps every card onto empty id maps, which is silent corruption worse than a missing example collection"
- (mt/with-model-cleanup [:model/Database]
- (mt/with-temp [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}]
- (let [recreate-called? (atom false)]
- (with-redefs [example-content/recreate-example-content! (fn [_] (reset! recreate-called? true))]
- (mt/with-dynamic-fn-redefs [sync/sync-database! (fn [_] (throw (ex-info "sync boom" {})))]
- (#'sample-data/update-sample-database-if-needed! old-sample)))
- (testing "recreate is skipped"
- (is (false? @recreate-called?)))
- (testing "the engine swap itself still completed - a new SQLite sample DB exists"
- (is (t2/exists? :model/Database :is_sample true :engine :sqlite))))))))
-
(defn- db-level-perms
"DB-level data-permission rows for `db-id` as a comparable map {[group-id perm-type] perm-value}."
[db-id]
diff --git a/test/metabase/warehouses/models/database_test.clj b/test/metabase/warehouses/models/database_test.clj
index 88f09d1a2d3..f1893bc8d9d 100644
--- a/test/metabase/warehouses/models/database_test.clj
+++ b/test/metabase/warehouses/models/database_test.clj
@@ -708,20 +708,6 @@
(testing ":write_data_details :auth-provider is keywordized"
(is (keyword? (get-in db [:write_data_details :auth-provider]))))))))
-(deftest user-may-not-update-sample-database-test
- (mt/with-temp [:model/Database {:keys [id] :as _sample-database} {:engine :h2
- :is_sample true
- :name "Sample Database"
- :details {:db "./resources/sample-database.db;USER=GUEST;PASSWORD=guest"}}]
- (testing " updating the engine of a sample database is not allowed"
- (is (thrown-with-msg?
- clojure.lang.ExceptionInfo
- #"The engine on a sample database cannot be changed."
- (t2/update! :model/Database id {:engine :sqlite}))))
- (testing " updating other attributes of a sample database is allowed"
- (t2/update! :model/Database id {:name "My New Name"})
- (is (= "My New Name" (t2/select-one-fn :name :model/Database :id id))))))
-
(driver/register! ::test, :abstract? true)
(deftest preserve-driver-namespaces-test