The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Replace H2 with SQLite for sample data
Ben Grabowebb8e4838b51
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex dd0901e0f1a..dfd626cd0d9 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1924,11 +1924,18 @@{:team "UX West":api #{metabase.sample-data.coremetabase.sample-data.init}- :uses #{plugins+ :uses #{config+ models+ pluginssettingssyncutil}- :model-imports #{:model/Database}}+ :model-imports #{:model/Card+ :model/Dashboard+ :model/DashboardCard+ :model/Database+ :model/Field+ :model/Table}}search{:team "UX West"diff --git a/.github/workflows/drivers.yml b/.github/workflows/drivers.ymlindex ecfd7b7bcb0..c6a82d31057 100644--- a/.github/workflows/drivers.yml+++ b/.github/workflows/drivers.yml@@ -1405,7 +1405,7 @@ jobs:- name: Fail on quarantine conflictif: ${{ needs.determine-driver-skips.outputs.sqlite-quarantine-conflict == 'true' }}run: |- echo "::error::SQLite driver is quarantined but you changed files in modules/drivers/sqlite/"+ echo "::error::SQLite driver is quarantined but you changed files that affect it"echo ""echo "Your changes will NOT be tested because the driver is currently quarantined."echo "To run SQLite tests, add the label: break-quarantine-sqlite"diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.ymlindex f151aa6d869..4b55e93ba89 100644--- a/.github/workflows/snyk.yml+++ b/.github/workflows/snyk.yml@@ -131,9 +131,6 @@ jobs:- name: Driver - SparkSQLfile: modules/drivers/sparksql/pom.xmlcategory: driver-sparksql- - name: Driver - SQLite- file: modules/drivers/sqlite/pom.xml- category: driver-sqlite- name: Driver - SQL Serverfile: modules/drivers/sqlserver/pom.xmlcategory: driver-sqlserverdiff --git a/.gitignore b/.gitignoreindex 570cad1100d..af7e0df5814 100644--- a/.gitignore+++ b/.gitignore@@ -92,6 +92,7 @@ cypress.env.json/resources/metabase/config/modules.edn/resources/namespaces.edn/resources/sample-dataset.db.trace.db+!/resources/sample-database.*/resources/version.properties/screenshots/sqlite.dbdiff --git a/bin/build-backend-for-test b/bin/build-backend-for-testindex 781a19e1d0a..4ca22592e79 100755--- a/bin/build-backend-for-test+++ b/bin/build-backend-for-test@@ -23,7 +23,7 @@ readonly JAR_PATH="target/uberjar/$JAR_FILENAME"backend-source-hash() {# hash all the files that might change a backend-only uberjar build(- find src deps.edn resources/sample-database.db.mv.db -type f -print0 | xargs -0 shasum ;+ find src deps.edn resources/sample-database.sqlite resources/sample-database.db.mv.db -type f -print0 | xargs -0 shasum ;find resources -type f \( -iname \*.clj -o -iname \*.edn -o -iname \*.yaml -o -iname \*.properties -o -iname \*.html \) -not -name "version.properties" -print0 | xargs -0 shasum ;) | shasum | awk '{ print $1 }'}diff --git a/bin/build/src/i18n/enumerate.clj b/bin/build/src/i18n/enumerate.cljindex 0b41f70d9c3..d4cd2926441 100644--- a/bin/build/src/i18n/enumerate.clj+++ b/bin/build/src/i18n/enumerate.clj@@ -26,7 +26,6 @@"/modules/drivers/redshift/src""/modules/drivers/snowflake/src""/modules/drivers/sparksql/src"- "/modules/drivers/sqlite/src""/modules/drivers/sqlserver/src""/modules/drivers/vertica/src"]))diff --git a/deps.edn b/deps.ednindex fcdd266c109..22fb92f7aaf 100644--- a/deps.edn+++ b/deps.edn@@ -182,6 +182,7 @@org.slf4j/slf4j-api {:mvn/version "2.0.17"} ; abstraction for logging frameworks -- allows end user to plug in desired logging framework at deployment timeorg.tcrawley/dynapath {:mvn/version "1.1.0"} ; Dynamically add Jars (e.g. Oracle or Vertica) to classpathorg.threeten/threeten-extra {:mvn/version "1.8.0"} ; extra Java 8 java.time classes like DayOfMonth and Quarter+ org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"} ; SQLite driver -- ships in core because the bundled Sample Database runs on itpotemkin/potemkin {:mvn/version "0.4.8" ; utility macros & fns:exclusions [riddley/riddley]}pretty/pretty {:mvn/version "1.0.5"} ; protocol for defining how custom types should be pretty printed@@ -506,7 +507,6 @@"modules/drivers/redshift/test""modules/drivers/snowflake/test""modules/drivers/sparksql/test"- "modules/drivers/sqlite/test""modules/drivers/sqlserver/test""modules/drivers/starburst/test""modules/drivers/vertica/test"]}@@ -575,8 +575,6 @@"modules/drivers/snowflake/test""modules/drivers/sparksql/src""modules/drivers/sparksql/test"- "modules/drivers/sqlite/src"- "modules/drivers/sqlite/test""modules/drivers/sqlserver/src""modules/drivers/sqlserver/test""modules/drivers/starburst/src"@@ -606,7 +604,6 @@"modules/drivers/redshift/src""modules/drivers/snowflake/src""modules/drivers/sparksql/src"- "modules/drivers/sqlite/src""modules/drivers/sqlserver/src""modules/drivers/starburst/src""modules/drivers/vertica/src"]@@ -631,7 +628,6 @@"modules/drivers/redshift/src""modules/drivers/snowflake/src""modules/drivers/sparksql/src"- "modules/drivers/sqlite/src""modules/drivers/sqlserver/src""modules/drivers/starburst/src""modules/drivers/vertica/src"]diff --git a/dev/src/dev/sample_database_convert.clj b/dev/src/dev/sample_database_convert.cljnew file mode 100644index 00000000000..d457447ecc5--- /dev/null+++ b/dev/src/dev/sample_database_convert.clj@@ -0,0 +1,225 @@+(ns dev.sample-database-convert+ "One-shot converter: read the H2 sample database checked in at+ `resources/sample-database.db.mv.db` and write an equivalent SQLite file to+ `resources/sample-database.sqlite`.++ Run via REPL: `(dev.sample-database-convert/convert!)`.++ This exists so the SQLite sample-DB POC has a reproducible derivation from+ the pre-existing H2 artifact. Once we stop using H2 in E2E tests and retire+ the H2 snapshot DB for good we should replace this with a generator that+ builds the dataset from source rather than re-deriving it from H2."+ (:require+ [clojure.java.io :as io]+ [clojure.string :as str]+ [honey.sql :as sql]+ [metabase.driver-api.core :as driver-api]+ [metabase.util.log :as log]+ [next.jdbc :as jdbc]+ [next.jdbc.prepare :as prepare]+ [next.jdbc.result-set :as rs])+ (:import+ (java.sql Clob Date ResultSetMetaData Time Timestamp Types)+ (java.time LocalDate LocalDateTime LocalTime OffsetDateTime ZonedDateTime)))++;; A Clob is a LOB handle tied to the open result set; `jdbc/execute!` with `rs/as-arrays` realizes rows for use+;; after the set is consumed, so decode it to a String at read time (while the handle is valid) rather than later in+;; `coerce-value`. Uses the same helper as the H2 driver; otherwise SQLite-JDBC stores the Clob's toString.+(extend-protocol rs/ReadableColumn+ Clob+ (read-column-by-label [^Clob v _] (driver-api/clob->str v))+ (read-column-by-index [^Clob v _ _] (driver-api/clob->str v)))++(set! *warn-on-reflection* true)++(def ^:private h2-source-path+ "resources/sample-database.db.mv.db")++(def ^:private sqlite-target-path+ "resources/sample-database.sqlite")++(defn- h2-db []+ ;; Read-only; do not touch the bundled artifact. H2 expects the dbname WITHOUT+ ;; the `.mv.db` suffix on disk.+ {:dbtype "h2"+ :dbname (str (str/replace (.getAbsolutePath (io/file h2-source-path)) #"\.mv\.db$" "")+ ";ACCESS_MODE_DATA=r;IFEXISTS=TRUE")})++(defn- sqlite-db []+ {:dbtype "sqlite"+ :dbname (.getAbsolutePath (io/file sqlite-target-path))})++(defn- list-tables [h2]+ (->> (jdbc/execute! h2 ["SELECT TABLE_SCHEMA, TABLE_NAME+ FROM INFORMATION_SCHEMA.TABLES+ WHERE TABLE_SCHEMA = 'PUBLIC'"]+ {:builder-fn rs/as-unqualified-maps})+ (map (juxt :TABLE_SCHEMA :TABLE_NAME))))++(def ^:private jdbc-type-code->sqlite-type+ "Fallback declared SQLite type by JDBC type code (`java.sql.Types`), used only when H2 returns a blank type name."+ {Types/INTEGER "INTEGER"+ Types/BIGINT "BIGINT"+ Types/SMALLINT "SMALLINT"+ Types/TINYINT "TINYINT"+ Types/BOOLEAN "BOOLEAN"+ Types/BIT "BOOLEAN"+ Types/REAL "REAL"+ Types/FLOAT "FLOAT"+ Types/DOUBLE "DOUBLE PRECISION"+ Types/DECIMAL "DECIMAL"+ Types/NUMERIC "NUMERIC"+ Types/DATE "DATE"+ Types/TIME "TIME"+ Types/TIMESTAMP "TIMESTAMP"+ Types/TIMESTAMP_WITH_TIMEZONE "TIMESTAMP"+ Types/BLOB "BLOB"+ Types/VARBINARY "BLOB"+ Types/BINARY "BLOB"})++(defn- h2-type->sqlite+ "Pass the H2 declared type name through verbatim. SQLite stores the declared+ type string in `sqlite_master` / `PRAGMA table_info` and Metabase's+ `database-type->base-type` mapping for SQLite is regex-based on that string+ (`#\"BIGINT\"` -> `:type/BigInteger`, `#\"CHAR\"` -> `:type/Text`,+ `#\"DOUB\"` -> `:type/Float`, etc.). Preserving H2's names means sync+ produces the same `:database_type` and `:base_type` values as the H2+ original, which keeps `resources/sample-content.edn` consistent without+ rewriting every field entry. JDBC type code is the disambiguator only when+ the H2 metadata returns a blank type name."+ [^long jdbc-type ^String h2-type-name]+ (if (str/blank? h2-type-name)+ (get jdbc-type-code->sqlite-type (int jdbc-type) "TEXT")+ h2-type-name))++(defn- column-defs+ "Return [{:name col-name :sqlite-type ... :jdbc-type long}] for the given+ schema.table by introspecting the result-set metadata of a 0-row query against H2."+ [h2 schema table]+ (with-open [conn (jdbc/get-connection h2)+ ps (.prepareStatement conn (format "SELECT * FROM \"%s\".\"%s\" LIMIT 0"+ schema table))+ rs (.executeQuery ps)]+ (let [^ResultSetMetaData md (.getMetaData rs)+ n (.getColumnCount md)]+ (vec+ (for [i (range 1 (inc n))]+ {:name (.getColumnName md i)+ :sqlite-type (h2-type->sqlite (.getColumnType md i)+ (.getColumnTypeName md i))+ :jdbc-type (.getColumnType md i)})))))++(defn- fk-defs+ "Return [{:fk-cols [...] :pk-table ... :pk-cols [...]}] for schema.table by+ reading H2's imported (foreign) keys. SQLite only records FK relationships+ when they are declared as FOREIGN KEY clauses in the CREATE TABLE DDL, and+ Metabase's sync derives `fk_target_field_id` (and thus implicit joins) from+ those declarations. Grouping on `fk_name` keeps composite keys intact."+ [h2 schema table]+ (with-open [conn (jdbc/get-connection h2)]+ (->> (resultset-seq (.getImportedKeys (.getMetaData conn) nil schema table))+ (group-by :fk_name)+ vals+ (mapv (fn [rows]+ (let [ordered (sort-by :key_seq rows)]+ {:fk-cols (mapv :fkcolumn_name ordered)+ :pk-table (:pktable_name (first ordered))+ :pk-cols (mapv :pkcolumn_name ordered)}))))))++(defn- pk-defs+ "Return the ordered primary-key column names for schema.table by reading H2's+ primary keys. Emitted as a table-level `PRIMARY KEY (...)` constraint (rather+ than an inline `INTEGER PRIMARY KEY`) so the declared column types — and thus+ Metabase's synced `:database_type`/`:base_type` — stay identical to H2. SQLite+ records the constraint in its catalog, so JDBC `getPrimaryKeys` reports it and+ sync assigns `:type/PK`, restoring ID-filter widgets and PK drills."+ [h2 schema table]+ (with-open [conn (jdbc/get-connection h2)]+ (->> (resultset-seq (.getPrimaryKeys (.getMetaData conn) nil schema table))+ (sort-by :key_seq)+ (mapv :column_name))))++(defn- create-table-sql [table cols fks pks]+ ;; Identifiers are simple uppercase H2 names (no dots), so plain `keyword` is safe; `:quoted true` double-quotes+ ;; them. The declared SQLite type is emitted raw because it is an arbitrary string (e.g. "DOUBLE PRECISION").+ (first+ (sql/format+ {:create-table (keyword table)+ :with-columns+ (concat+ (for [{:keys [name sqlite-type]} cols]+ [(keyword name) [:raw sqlite-type]])+ (when (seq pks)+ [[(into [:primary-key] (map keyword) pks)]])+ (for [{:keys [fk-cols pk-table pk-cols]} fks]+ [(into [:foreign-key] (map keyword) fk-cols)+ (into [:references (keyword pk-table)] (map keyword) pk-cols)]))}+ {:dialect :ansi :quoted true})))++(defn- insert-sql [table cols]+ ;; `[:raw "?"]` forces a positional placeholder per column for the batched PreparedStatement (a nil value would+ ;; render as NULL instead). The actual values are bound later via `.setObject`/`.setNull`.+ (first+ (sql/format+ {:insert-into (keyword table)+ :columns (mapv (comp keyword :name) cols)+ :values [(vec (repeat (count cols) [:raw "?"]))]}+ {:dialect :ansi :quoted true})))++(defn- select-sql [schema table cols]+ (first+ (sql/format+ {:select (mapv (comp keyword :name) cols)+ :from [(keyword (str schema "." table))]}+ {:dialect :ansi :quoted true})))++(defn- coerce-value+ "Coerce H2 values to types SQLite-JDBC accepts cleanly."+ [v]+ (cond+ (nil? v) nil+ (instance? LocalDate v) (str v)+ (instance? LocalDateTime v) (str v)+ (instance? LocalTime v) (str v)+ (instance? OffsetDateTime v) (str v)+ (instance? ZonedDateTime v) (str v)+ (instance? Timestamp v) (str (.toLocalDateTime ^Timestamp v))+ (instance? Date v) (str (.toLocalDate ^Date v))+ (instance? Time v) (str (.toLocalTime ^Time v))+ :else v))++(defn- copy-table!+ [h2 sqlite schema table]+ (let [cols (column-defs h2 schema table)+ fks (fk-defs h2 schema table)+ pks (pk-defs h2 schema table)+ ddl (create-table-sql table cols fks pks)+ ins-sql (insert-sql table cols)]+ (log/info " Creating table" table "with" (count cols) "columns")+ (jdbc/execute! sqlite [ddl])+ (let [rows (->> (jdbc/execute! h2 [(select-sql schema table cols)] {:builder-fn rs/as-arrays})+ rest ; Skip column header+ (mapv #(mapv coerce-value %)))]+ (jdbc/with-transaction [tx sqlite]+ (with-open [ps (jdbc/prepare tx [ins-sql])]+ (prepare/execute-batch! ps rows {:batch-size 5000})))+ (log/info " Wrote" (count rows) "rows to" table))))++(defn convert!+ "Run the H2 -> SQLite conversion. Deletes any existing SQLite file first."+ []+ (let [target (io/file sqlite-target-path)]+ (when (.exists target)+ (log/info "Deleting existing" sqlite-target-path)+ (.delete target)))+ (log/info "Converting H2 sample DB ->" sqlite-target-path)+ (let [h2 (h2-db)+ sqlite (sqlite-db)+ tables (list-tables h2)]+ (log/info "Found" (count tables) "tables in PUBLIC schema")+ (doseq [[schema table] tables]+ (copy-table! h2 sqlite schema table))+ (log/info "Done. Output:" (.getAbsolutePath (io/file sqlite-target-path)))))++(comment+ (convert!))diff --git a/e2e/runner/cypress-runner-backend.js b/e2e/runner/cypress-runner-backend.jsindex 2062f39d3e4..351ce44db8c 100755--- a/e2e/runner/cypress-runner-backend.js+++ b/e2e/runner/cypress-runner-backend.js@@ -23,6 +23,10 @@ process.env.MB_CUSTOM_VIZ_PLUGIN_DEV_MODE_ENABLED =// Expose the `en-ZZ` pseudo-locale in the Cypress backend's language pickers and API validation.process.env.MB_ENABLE_TEST_LOCALES = "true";+// Use the H2 sample database in E2E tests. Production ships SQLite, but the test suite was written+// against the H2 sample data; this defers migrating the tests to SQLite until after the release.+process.env.MB_SAMPLE_DATABASE_ENGINE = "h2";+if (!process.env.CI) {// Use a temporary copy of the sample db so it won't use and lock the db used for local developmentprocess.env.MB_INTERNAL_DO_NOT_USE_SAMPLE_DB_DIR = path.resolve(diff --git a/e2e/test/scenarios/admin/databases.cy.spec.js b/e2e/test/scenarios/admin/databases.cy.spec.jsindex d83e142b1e6..5dbe9b0e5bc 100644--- a/e2e/test/scenarios/admin/databases.cy.spec.js+++ b/e2e/test/scenarios/admin/databases.cy.spec.js@@ -532,9 +532,7 @@ describe("scenarios > admin > databases > exceptions", () => {.findByRole("button", { name: "Edit connection details" }).should("be.disabled").trigger("mouseenter", { force: true });- H.tooltip().findByText(- "This database is managed by Metabase Cloud and cannot be modified.",- );+ H.tooltip().findByText("The sample database cannot be edited.");cy.findByTestId("database-actions-panel").should("not.exist");});@@ -654,139 +652,6 @@ describe("scenarios > admin > databases > sample database", () => {beforeEach(() => {H.restore();cy.signInAsAdmin();- cy.intercept("PUT", "/api/database/*").as("databaseUpdate");- });-- it("database settings", () => {- visitDatabase(SAMPLE_DB_ID);-- cy.findAllByTestId("database-connection-info-section").should(- "contain.text",- "Connected",- );-- editDatabase();-- // should not display a setup help card- // eslint-disable-next-line metabase/no-unscoped-text-selectors -- deprecated usage- cy.findByText("Need help connecting?").should("not.exist");-- cy.log(- "should not be possible to change database type for the Sample Database (metabase#16382)",- );- cy.findByLabelText("Database type")- .should("have.value", "H2")- .and("be.disabled");-- cy.log("should correctly display connection settings");- cy.findByLabelText("Display name").should("have.value", "Sample Database");- cy.findByLabelText("Connection String")- .should("have.attr", "value")- .and("contain", "sample-database.db");-- cy.log("should be possible to modify the connection settings");- // eslint-disable-next-line metabase/no-unscoped-text-selectors -- deprecated usage- cy.findByText("Show advanced options").click();- // `auto_run_queries` toggle should be ON by default- cy.findByLabelText(/Rerun queries for simple explorations/)- .should("have.attr", "data-checked", "true")- .click();- // Reported failing in v0.36.4- cy.log(- "should respect the settings for automatic query running (metabase#13187)",- );- cy.findByLabelText(/Rerun queries for simple explorations/).should(- "have.attr",- "data-checked",- "false",- );-- cy.log("change the metadata_sync period");- cy.findByLabelText(/Choose when syncs and scans happen/).click();- cy.findByDisplayValue("Hourly").click();- H.popover().within(() => {- cy.findByText("Daily").click();- });-- // "lets you change the cache_field_values period"- cy.findByDisplayValue("Never, I'll do this manually if I need to")- .should("be.visible")- .click();-- H.popover().findByText("Regularly, on a schedule").click();- cy.findAllByDisplayValue("Daily").should("have.length", 2).eq(1).click();- H.popover().findByText("Weekly").click();-- cy.button("Save changes").click();- cy.wait("@databaseUpdate").then(({ response: { body } }) => {- editDatabase();- expect(body.details["let-user-control-scheduling"]).to.equal(true);- expect(body.schedules.metadata_sync.schedule_type).to.equal("daily");- expect(body.schedules.cache_field_values.schedule_type).to.equal(- "weekly",- );- });-- // "lets you change the cache_field_values to 'Only when adding a new filter widget'"- cy.findByDisplayValue("Regularly, on a schedule").click();- H.popover().findByText("Only when adding a new filter widget").click();- cy.button("Save changes", { timeout: 10000 }).click();- cy.wait("@databaseUpdate").then(({ response: { body } }) => {- editDatabase();- expect(body.is_full_sync).to.equal(false);- expect(body.is_on_demand).to.equal(true);- });-- // and back to never- cy.findByDisplayValue("Only when adding a new filter widget").click();- H.popover().findByText("Never, I'll do this manually if I need to").click();- cy.button("Save changes", { timeout: 10000 }).click();- cy.wait("@databaseUpdate").then(({ response: { body } }) => {- editDatabase();- expect(body.is_full_sync).to.equal(false);- expect(body.is_on_demand).to.equal(false);- });- });-- it("allows to save the default schedule (metabase#57198)", () => {- visitDatabase(SAMPLE_DB_ID);- editDatabase();- cy.findByRole("button", { name: /Show advanced options/ }).click();- cy.findByLabelText(/Choose when syncs and scans happen/).click({- force: true,- });- cy.button("Save changes").click();- cy.wait("@databaseUpdate").then(({ request: { body }, response }) => {- expect(body.is_full_sync).to.equal(false);- expect(body.is_on_demand).to.equal(false);- // frontend sends wrong value but backend automatically corrects it for us:- expect(response.body.schedules.cache_field_values).to.equal(null);- });-- editDatabase();- cy.findByDisplayValue("Never, I'll do this manually if I need to").click();- H.popover().findByText("Regularly, on a schedule").click();- cy.button("Save changes").click();- cy.wait("@databaseUpdate").then(({ request: { body } }) => {- expect(body.is_full_sync).to.equal(true);- expect(body.is_on_demand).to.equal(false);- expect(body.schedules.cache_field_values).to.deep.eq({- schedule_day: "mon",- schedule_frame: null,- schedule_hour: 0,- schedule_type: "daily",- });- });-- editDatabase();- cy.findByDisplayValue("Regularly, on a schedule").click();- H.popover().findByText("Only when adding a new filter widget").click();- cy.button("Save changes").click();- cy.wait("@databaseUpdate").then(({ request: { body } }) => {- expect(body.is_full_sync).to.equal(false);- expect(body.is_on_demand).to.equal(true);- expect(body.schedules.cache_field_values).to.equal(null);- });});it("database actions", () => {diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/sample_database_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/sample_database_test.cljnew file mode 100644index 00000000000..883527e6d1c--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/sandbox/sample_database_test.clj@@ -0,0 +1,137 @@+(ns metabase-enterprise.sandbox.sample-database-test+ "Sandboxing (row and column security) against the bundled SQLite Sample Database.++ The shared sandbox helpers in [[metabase-enterprise.sandbox.test-util]] run against the `test-data`+ dataset, not the real Sample Database that ships with Metabase. These tests load the actual bundled+ Sample Database, sync it, and exercise sandboxing end-to-end so we can confirm row/column security still+ works after the H2 -> SQLite engine swap.++ [[doc-walkthrough-row-filter-test]] and [[doc-walkthrough-custom-view-test]] follow the customer-facing+ walkthrough in docs/permissions/row-and-column-security-examples.md step for step, so a failure here means+ the documented path no longer works on the shipped sample data."+ (:require+ [clojure.test :refer :all]+ [metabase.permissions.models.data-permissions :as data-perms]+ [metabase.query-processor :as qp]+ [metabase.request.core :as request]+ [metabase.sample-data.impl :as sample-data]+ [metabase.sync.core :as sync]+ [metabase.test :as mt]+ [toucan2.core :as t2]))++(defn- do-with-sample-database+ "Call `thunk` with a freshly extracted and synced bundled Sample Database, with the `:sandboxes` premium+ feature enabled."+ [thunk]+ (mt/with-premium-features #{:sandboxes}+ (mt/with-temp [:model/Database db {:details (#'sample-data/try-to-extract-sample-database! :sqlite)+ :engine :sqlite+ :name "Sample Database"}]+ (sync/sync-database! db)+ (thunk db))))++(defn- table-id [db table-name]+ (t2/select-one-pk :model/Table :db_id (:id db) :name table-name))++(defn- field-id [db table-name field-name]+ (t2/select-one-pk :model/Field :table_id (table-id db table-name) :name field-name))++(defn- rows [query]+ (mt/rows (qp/process-query query)))++(defn- col-index [result col-name]+ (first (keep-indexed (fn [i col] (when (= (:name col) col-name) i))+ (get-in result [:data :cols]))))++;; Cloyd Beer is the example customer used throughout the docs; 2499 is his ID in the People table.+(def ^:private cloyd-user-id 2499)++(deftest baseline-no-sandbox-test+ (testing "Sanity check: without sandboxing, an admin sees the whole Sample Database"+ (do-with-sample-database+ (fn [db]+ (is (= 18760+ (ffirst (rows {:database (:id db) :type :query+ :query {:source-table (table-id db "ORDERS") :aggregation [[:count]]}}))))+ (is (= 2500+ (ffirst (rows {:database (:id db) :type :query+ :query {:source-table (table-id db "PEOPLE") :aggregation [[:count]]}}))))))))++(deftest doc-walkthrough-row-filter-test+ (testing "docs/permissions/row-and-column-security-examples.md — 'Filtering rows based on user attributes'"+ (do-with-sample-database+ (fn [db]+ (let [orders-id (table-id db "ORDERS")+ people-id (table-id db "PEOPLE")+ orders-uid (field-id db "ORDERS" "USER_ID")+ people-pk (field-id db "PEOPLE" "ID")]+ (mt/with-temp+ ;; "Create a group called Customers" + "Create a user account for Cloyd Beer" with the+ ;; user_id: 2499 attribute, and add him to the group.+ [:model/PermissionsGroup group {:name "Customers"}+ :model/User user {:login_attributes {"user_id" cloyd-user-id}}+ :model/PermissionsGroupMembership _ {:group_id (:id group) :user_id (:id user)}+ ;; "Set View data -> Row and column security ... Filter by a column on this table." for Orders+ ;; (USER_ID) and People (ID), each = the user_id attribute.+ :model/Sandbox _ {:group_id (:id group)+ :table_id orders-id+ :attribute_remappings {"user_id" [:dimension [:field orders-uid nil]]}}+ :model/Sandbox _ {:group_id (:id group)+ :table_id people-id+ :attribute_remappings {"user_id" [:dimension [:field people-pk nil]]}}]+ ;; "Block permissions for the All users group" + grant the Customers group granular access.+ (mt/with-no-data-perms-for-all-users!+ (data-perms/set-database-permission! group (:id db) :perms/view-data :unrestricted)+ (data-perms/set-table-permission! group orders-id :perms/create-queries :query-builder)+ (data-perms/set-table-permission! group people-id :perms/create-queries :query-builder)+ (request/with-current-user (:id user)+ (testing "Orders is filtered to only Cloyd's rows"+ (is (= 17+ (ffirst (rows {:database (:id db) :type :query+ :query {:source-table orders-id :aggregation [[:count]]}}))))+ (is (= [cloyd-user-id]+ (map first (rows {:database (:id db) :type :query+ :query {:source-table orders-id+ :breakout [[:field orders-uid nil]]}})))))+ (testing "People is filtered to only Cloyd's row"+ (is (= [cloyd-user-id]+ (map first (rows {:database (:id db) :type :query+ :query {:source-table people-id}})))))))))))))++(deftest doc-walkthrough-custom-view-test+ (testing "docs/permissions/row-and-column-security-examples.md — 'Custom example 2: Filtering rows and columns'"+ (do-with-sample-database+ (fn [db]+ (let [orders-id (table-id db "ORDERS")]+ (mt/with-temp+ [:model/PermissionsGroup group {:name "Customers"}+ :model/User user {:login_attributes {"user_id" cloyd-user-id}}+ :model/PermissionsGroupMembership _ {:group_id (:id group) :user_id (:id user)}+ ;; "Create a SQL question with a variable" selecting a subset of columns, WHERE user_id = {{user_id}}.+ :model/Card {card-id :id} {:database_id (:id db)+ :dataset_query+ {:database (:id db)+ :type :native+ :native {:query (str "SELECT id, created_at, product_id, quantity, total, user_id "+ "FROM orders WHERE user_id = {{user_id}}")+ :template-tags {"user_id" {:name "user_id"+ :display-name "User ID"+ :type :number}}}}}+ ;; "Use a saved question to create a custom view" + map {{user_id}} to the user_id attribute.+ :model/Sandbox _ {:group_id (:id group)+ :table_id orders-id+ :card_id card-id+ :attribute_remappings {"user_id" [:variable [:template-tag "user_id"]]}}]+ (mt/with-no-data-perms-for-all-users!+ (data-perms/set-database-permission! group (:id db) :perms/view-data :unrestricted)+ (data-perms/set-table-permission! group orders-id :perms/create-queries :query-builder)+ (request/with-current-user (:id user)+ (let [result (qp/process-query {:database (:id db) :type :query+ :query {:source-table orders-id}})]+ (testing "only the columns from the custom view are exposed"+ (is (= ["ID" "USER_ID" "PRODUCT_ID" "TOTAL" "CREATED_AT" "QUANTITY"]+ (mapv :name (get-in result [:data :cols])))))+ (testing "rows are filtered to only Cloyd's orders"+ (is (= 17 (:row_count result)))+ (is (= [cloyd-user-id]+ (distinct (map #(nth % (col-index result "USER_ID")) (mt/rows result)))))))))))))))diff --git a/frontend/src/metabase-types/api/settings.ts b/frontend/src/metabase-types/api/settings.tsindex b774b91e6d6..6d3aade894d 100644--- a/frontend/src/metabase-types/api/settings.ts+++ b/frontend/src/metabase-types/api/settings.ts@@ -112,6 +112,7 @@ export interface Engine {"details-fields"?: DatabaseFieldOrGroup[];source: EngineSource;"superseded-by": string | null;+ "creatable?"?: boolean;"extra-info": {"db-routing-info": {text: string;diff --git a/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.tsx b/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.tsxindex d930a591189..1aea550db01 100644--- a/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.tsx+++ b/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.tsx@@ -10,7 +10,10 @@ import {} from "metabase/api";import { listTag } from "metabase/api/tags";import { ActionButton } from "metabase/common/components/ActionButton";-import { isDbModifiable } from "metabase/common/utils/database";+import {+ getDbNotModifiableMessage,+ isDbModifiable,+} from "metabase/common/utils/database";import { useDispatch } from "metabase/redux";import { Button, Flex, Tooltip } from "metabase/ui";import { isSyncCompleted } from "metabase/utils/syncing";@@ -62,7 +65,7 @@ export const DatabaseConnectionInfoSection = ({<DatabaseConnectionHealthInfo databaseId={database.id} /><Tooltipdisabled={isDbModifiable(database)}- label={t`This database is managed by Metabase Cloud and cannot be modified.`}+ label={getDbNotModifiableMessage(database)}><ButtononClick={openDbDetailsModal}diff --git a/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.unit.spec.tsx b/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.unit.spec.tsxindex 5677db6f09e..2a2f14ce09e 100644--- a/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.unit.spec.tsx+++ b/frontend/src/metabase/admin/databases/components/DatabaseConnectionInfoSection/DatabaseConnectionInfoSection.unit.spec.tsx@@ -189,4 +189,13 @@ describe("DatabaseConnectionInfoSection", () => {).not.toBeInTheDocument();});});++ describe("Sample database", () => {+ it("should disable editing the connection details", async () => {+ setup({ database: createMockDatabase({ is_sample: true }) });+ expect(+ await screen.findByRole("button", { name: /Edit connection details/i }),+ ).toBeDisabled();+ });+ });});diff --git a/frontend/src/metabase/admin/databases/components/DatabaseEditConnectionForm.tsx b/frontend/src/metabase/admin/databases/components/DatabaseEditConnectionForm.tsxindex 7995e2bc52d..de7722e5f1b 100644--- a/frontend/src/metabase/admin/databases/components/DatabaseEditConnectionForm.tsx+++ b/frontend/src/metabase/admin/databases/components/DatabaseEditConnectionForm.tsx@@ -2,7 +2,6 @@ import type { LocationDescriptorObject } from "history";import { updateIn } from "icepick";import { type ComponentType, useState } from "react";import { type Route, withRouter } from "react-router";-import { t } from "ttag";import _ from "underscore";import ErrorBoundary from "metabase/ErrorBoundary";@@ -10,7 +9,10 @@ import { GenericError } from "metabase/common/components/ErrorPages";import { LeaveRouteConfirmModal } from "metabase/common/components/LeaveConfirmModal";import { LoadingAndErrorWrapper } from "metabase/common/components/LoadingAndErrorWrapper";import { useCallbackEffect } from "metabase/common/hooks/use-callback-effect";-import { isDbModifiable } from "metabase/common/utils/database";+import {+ getDbNotModifiableMessage,+ isDbModifiable,+} from "metabase/common/utils/database";import { DatabaseForm } from "metabase/databases/components/DatabaseForm";import type {DatabaseFormConfig,@@ -95,6 +97,7 @@ export const DatabaseEditConnectionForm = withRouter({isDbModifiable({id: database?.id,is_attached_dwh: isAttachedDWH,+ is_sample: database?.is_sample,}) ? (<DatabaseForminitialValues={database}@@ -107,7 +110,7 @@ export const DatabaseEditConnectionForm = withRouter(onEngineChange={onEngineChange}/>) : (- <Text my="md">{t`This database is managed by Metabase Cloud and cannot be modified.`}</Text>+ <Text my="md">{getDbNotModifiableMessage(database)}</Text>)}</LoadingAndErrorWrapper><LeaveRouteConfirmModaldiff --git a/frontend/src/metabase/admin/databases/components/DatabaseEditConnectionForm.unit.spec.tsx b/frontend/src/metabase/admin/databases/components/DatabaseEditConnectionForm.unit.spec.tsxnew file mode 100644index 00000000000..098ce548760--- /dev/null+++ b/frontend/src/metabase/admin/databases/components/DatabaseEditConnectionForm.unit.spec.tsx@@ -0,0 +1,64 @@+import { Route } from "react-router";++import { mockSettings } from "__support__/settings";+import { renderWithProviders, screen } from "__support__/ui";+import { createMockState } from "metabase/redux/store/mocks";+import type { DatabaseData } from "metabase-types/api";+import { createMockEngines } from "metabase-types/api/mocks";++import { DatabaseEditConnectionForm } from "./DatabaseEditConnectionForm";++interface SetupOpts {+ database?: Partial<DatabaseData>;+ isAttachedDWH?: boolean;+}++function setup({ database, isAttachedDWH = false }: SetupOpts = {}) {+ const storeInitialState = createMockState({+ settings: mockSettings({ engines: createMockEngines() }),+ });++ renderWithProviders(+ <Route+ path="/"+ component={(routerProps) => (+ <DatabaseEditConnectionForm+ {...routerProps}+ database={database}+ isAttachedDWH={isAttachedDWH}+ onSubmitted={jest.fn()}+ onCancel={jest.fn()}+ formLocation="admin"+ />+ )}+ />,+ { withRouter: true, storeInitialState },+ );+}++describe("DatabaseEditConnectionForm", () => {+ it("renders the editable connection form for an ordinary database", () => {+ setup({ database: { id: 1, engine: "postgres" } });++ expect(screen.getByTestId("database-form")).toBeInTheDocument();+ expect(screen.queryByText(/cannot be edited/i)).not.toBeInTheDocument();+ });++ it("replaces the form with an explanation for the sample database", () => {+ setup({ database: { id: 1, is_sample: true, engine: "postgres" } });++ expect(screen.queryByTestId("database-form")).not.toBeInTheDocument();+ expect(+ screen.getByText("The sample database cannot be edited."),+ ).toBeInTheDocument();+ });++ it("replaces the form with an explanation for an attached DWH", () => {+ setup({ database: { id: 1, engine: "postgres" }, isAttachedDWH: true });++ expect(screen.queryByTestId("database-form")).not.toBeInTheDocument();+ expect(+ screen.getByText(/managed by Metabase Cloud and cannot be modified/i),+ ).toBeInTheDocument();+ });+});diff --git a/frontend/src/metabase/common/utils/database.ts b/frontend/src/metabase/common/utils/database.tsindex 9e16fdf12f0..4e4b4211011 100644--- a/frontend/src/metabase/common/utils/database.ts+++ b/frontend/src/metabase/common/utils/database.ts@@ -1,3 +1,5 @@+import { t } from "ttag";+import type {Card,Dashboard,@@ -7,9 +9,30 @@ import type {} from "metabase-types/api";export const isDbModifiable = (- database: { id?: DatabaseId; is_attached_dwh?: boolean } | undefined,+ database:+ | { id?: DatabaseId; is_attached_dwh?: boolean; is_sample?: boolean }+ | undefined,+) => {+ return !(+ database?.id != null &&+ (database.is_attached_dwh || database.is_sample)+ );+};++/**+ * Message explaining why a non-modifiable database cannot be edited. Only+ * meaningful when [[isDbModifiable]] returns false for the same database.+ * The cloud-managed message names Metabase Cloud literally (not the+ * whitelabeled name) so admins can tell platform-managed databases apart from+ * ones managed by their own whitelabeled instance.+ */+export const getDbNotModifiableMessage = (+ database: { is_sample?: boolean } | undefined,) => {- return !(database?.id != null && database.is_attached_dwh);+ return database?.is_sample+ ? t`The sample database cannot be edited.`+ : // eslint-disable-next-line metabase/no-literal-metabase-strings -- admin-only: must name Metabase Cloud to distinguish it from a whitelabeled instance+ t`This database is managed by Metabase Cloud and cannot be modified.`;};export const hasFeature = (diff --git a/frontend/src/metabase/databases/components/DatabaseForm/tests/schedule.unit.spec.tsx b/frontend/src/metabase/databases/components/DatabaseForm/tests/schedule.unit.spec.tsxnew file mode 100644index 00000000000..2c25b1bd2f0--- /dev/null+++ b/frontend/src/metabase/databases/components/DatabaseForm/tests/schedule.unit.spec.tsx@@ -0,0 +1,138 @@+import { within } from "@testing-library/react";+import userEvent from "@testing-library/user-event";++import { screen, waitFor } from "__support__/ui";++import { setup } from "./setup";++const fillConnection = async () => {+ await userEvent.type(screen.getByLabelText("Display name"), "My DB");+ await userEvent.type(+ screen.getByLabelText("Connection String"),+ "file:/somewhere",+ );+};++const openAdvancedOptions = async () => {+ await userEvent.click(screen.getByText("Show advanced options"));+};++// Enabling "Choose when syncs and scans happen" (let-user-control-scheduling) flips is_full_sync to false, which+// reveals the sync/cache schedule controls in their "none" (Never) state.+const enableScheduling = async () => {+ await userEvent.click(+ screen.getByLabelText(/Choose when syncs and scans happen/),+ );+};++const chooseScheduleMode = async (current: string, next: string) => {+ await userEvent.click(await screen.findByDisplayValue(current));+ const listbox = await screen.findByRole("listbox");+ await userEvent.click(within(listbox).getByText(next));+};++const save = async () => {+ const saveButton = screen.getByRole("button", { name: "Save" });+ await waitFor(() => expect(saveButton).toBeEnabled());+ await userEvent.click(saveButton);+};++describe("DatabaseForm scheduling", () => {+ it("submits auto_run_queries=false when 'Rerun queries' is turned off (metabase#13187)", async () => {+ const { onSubmit } = setup();+ await fillConnection();+ await openAdvancedOptions();++ await userEvent.click(+ screen.getByLabelText(/Rerun queries for simple explorations/),+ );++ await save();+ await waitFor(() => {+ expect(onSubmit).toHaveBeenCalledWith(+ expect.objectContaining({ auto_run_queries: false }),+ );+ });+ });++ it("submits is_full_sync=false when the user takes control of scheduling", async () => {+ const { onSubmit } = setup();+ await fillConnection();+ await openAdvancedOptions();++ await enableScheduling();++ await save();+ await waitFor(() => {+ expect(onSubmit).toHaveBeenCalledWith(+ expect.objectContaining({ is_full_sync: false, is_on_demand: false }),+ );+ });+ });++ it("submits is_on_demand=true for 'Only when adding a new filter widget' (metabase#57198)", async () => {+ const { onSubmit } = setup();+ await fillConnection();+ await openAdvancedOptions();+ await enableScheduling();++ await chooseScheduleMode(+ "Never, I'll do this manually if I need to",+ "Only when adding a new filter widget",+ );++ await save();+ await waitFor(() => {+ expect(onSubmit).toHaveBeenCalledWith(+ expect.objectContaining({ is_full_sync: false, is_on_demand: true }),+ );+ });+ });++ it("submits a daily cache schedule for 'Regularly, on a schedule' (metabase#57198)", async () => {+ const { onSubmit } = setup();+ await fillConnection();+ await openAdvancedOptions();+ await enableScheduling();++ await chooseScheduleMode(+ "Never, I'll do this manually if I need to",+ "Regularly, on a schedule",+ );++ await save();+ await waitFor(() => {+ expect(onSubmit).toHaveBeenCalledWith(+ expect.objectContaining({+ is_full_sync: true,+ is_on_demand: false,+ schedules: expect.objectContaining({+ cache_field_values: expect.objectContaining({+ schedule_type: "daily",+ }),+ }),+ }),+ );+ });+ });++ it("renders the saved cache schedule mode for an existing database (round-trip)", async () => {+ setup({+ initialValues: {+ id: 1,+ name: "My DB",+ details: {+ db: "file:/somewhere",+ "let-user-control-scheduling": true,+ },+ is_full_sync: false,+ is_on_demand: true,+ },+ });+ await openAdvancedOptions();++ expect(+ await screen.findByDisplayValue("Only when adding a new filter widget"),+ ).toBeInTheDocument();+ });+});diff --git a/frontend/src/metabase/databases/utils/engine.ts b/frontend/src/metabase/databases/utils/engine.tsindex 7529a506fc5..e68d7effdb8 100644--- a/frontend/src/metabase/databases/utils/engine.ts+++ b/frontend/src/metabase/databases/utils/engine.ts@@ -24,8 +24,9 @@ const isEngineVisible = (const isSelected = engineKey === selectedEngineKey;const isSuperseded = engine["superseded-by"] != null;const isSuperseding = engine["superseded-by"] === selectedEngineKey;+ const isCreatable = engine["creatable?"] !== false;- return isSelected || !isSuperseded || isSuperseding;+ return isSelected || ((!isSuperseded || isSuperseding) && isCreatable);};const getEngineOption = (engineKey: string, engine: Engine) => {diff --git a/frontend/src/metabase/databases/utils/engine.unit.spec.ts b/frontend/src/metabase/databases/utils/engine.unit.spec.tsindex 1976cf3f290..6a4030089d8 100644--- a/frontend/src/metabase/databases/utils/engine.unit.spec.ts+++ b/frontend/src/metabase/databases/utils/engine.unit.spec.ts@@ -3,6 +3,7 @@ import type { Engine } from "metabase-types/api";import {formatNativeQuery,getEngineNativeType,+ getEngineOptions,getNativeQueryLanguage,isDeprecatedEngine,} from "./engine";@@ -106,3 +107,34 @@ describe("isDeprecatedEngine", () => {expect(isDeprecatedEngine(engines, "buzzzzz")).toBe(false);});});++describe("getEngineOptions", () => {+ const engines: Record<string, Engine> = {+ postgres: {+ "driver-name": "PostgreSQL",+ source: { type: "official", contact: null },+ "superseded-by": null,+ "extra-info": null,+ },+ sqlite: {+ "driver-name": "SQLite",+ source: { type: "official", contact: null },+ "superseded-by": null,+ "creatable?": false,+ "extra-info": null,+ },+ };++ it("hides a non-creatable engine from the list", () => {+ const values = getEngineOptions(engines).map((option) => option.value);+ expect(values).toContain("postgres");+ expect(values).not.toContain("sqlite");+ });++ it("still shows a non-creatable engine when it is the selected engine", () => {+ const values = getEngineOptions(engines, "sqlite").map(+ (option) => option.value,+ );+ expect(values).toContain("sqlite");+ });+});diff --git a/mage/src/mage/modules.clj b/mage/src/mage/modules.cljindex 6a2da90ca06..d93c9b21cf5 100644--- a/mage/src/mage/modules.clj+++ b/mage/src/mage/modules.clj@@ -298,7 +298,6 @@"redshift" [:redshift]"snowflake" [:snowflake]"sparksql" [:sparksql]- "sqlite" [:sqlite]"sqlserver" [:sqlserver];; starburst tests are currently disabled in drivers.yml;; "starburst" [:starburst]diff --git a/modules/drivers/deps.edn b/modules/drivers/deps.ednindex b23239e9ae5..42f4e6c96ec 100644--- a/modules/drivers/deps.edn+++ b/modules/drivers/deps.edn@@ -19,7 +19,6 @@metabase/redshift {:local/root "redshift"}metabase/snowflake {:local/root "snowflake"}metabase/sparksql {:local/root "sparksql"}- metabase/sqlite {:local/root "sqlite"}metabase/sqlserver {:local/root "sqlserver"}metabase/starburst {:local/root "starburst"}metabase/vertica {:local/root "vertica"}}}diff --git a/modules/drivers/sqlite/deps.edn b/modules/drivers/sqlite/deps.edndeleted file mode 100644index b5be589a457..00000000000--- a/modules/drivers/sqlite/deps.edn+++ /dev/null@@ -1,5 +0,0 @@-{:paths- ["src" "resources"]-- :deps- {org.xerial/sqlite-jdbc {:mvn/version "3.50.3.0"}}}diff --git a/modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yaml b/modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yamldeleted file mode 100644index d87fca03246..00000000000--- a/modules/drivers/sqlite/resources/metabase/sqlite/metabase-plugin.yaml+++ /dev/null@@ -1,22 +0,0 @@-info:- name: Metabase SQLite Driver- version: 1.0.0-SNAPSHOT-3.25.2- description: Allows Metabase to connect to SQLite databases.-driver:- name: sqlite- display-name: SQLite- lazy-load: true- parent: sql-jdbc- connection-properties:- - name: db- display-name: Filename- placeholder: /path/to/toucan_sightings.sqlite- required: true- - cloud-ip-address-info- - advanced-options-start- - default-advanced-options-init:- - step: load-namespace- namespace: metabase.driver.sqlite- - step: register-jdbc-driver- class: org.sqlite.JDBCdiff --git a/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml b/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yamlnew file mode 100644index 00000000000..603f5c154a1--- /dev/null+++ b/resources/migrations/063/20260608_sqlite_sample_database_downgrade.yaml@@ -0,0 +1,13 @@+## On downgrade from a SQLite-sample-database version to an H2-sample-database version, remove the+## SQLite sample database and the content left dangling by it. Does nothing on rollforward; the+## H2 -> SQLite upgrade replacement is handled at startup in metabase.sample-data.impl.++databaseChangeLog:+ - changeSet:+ id: v63.2026-06-08T00:00:00+ author: bgrabow+ comment: Remove the SQLite sample database (and content it leaves empty) on downgrade to an H2 version+ validCheckSum: ANY+ changes:+ - customChange:+ class: "metabase.app_db.custom_migrations.MigrateAwayFromSqliteSampleDatabaseOnDowngrade"diff --git a/resources/sample-content.edn b/resources/sample-content.ednindex 8b2b6c11313..1dd9dffad52 100644--- a/resources/sample-content.edn+++ b/resources/sample-content.edn@@ -19,7 +19,7 @@:is_sample true,:id 1,:is_on_demand false,- :engine "h2",+ :engine "sqlite",:initial_sync_status "complete",:is_audit false,:dbms_version "{}",@@ -94,7 +94,7 @@:metabase_table({:description "Information on the user accounts registered with Sample Company.",:entity_type "entity/UserTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "PEOPLE",@@ -113,7 +113,7 @@:points_of_interest "Is it? You tell us!"}{:description "Confirmed Sample Company orders for a product, from a user.",:entity_type "entity/TransactionTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "ORDERS",@@ -133,7 +133,7 @@:points_of_interest "Is it? You tell us!"}{:description "Includes a catalog of all the products ever sold by the famed Sample Company.",:entity_type "entity/ProductTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "PRODUCTS",@@ -152,7 +152,7 @@:points_of_interest "Is it? You tell us!"}{:description "Reviews that Sample Company customers have left on our products.",:entity_type "entity/GenericTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "REVIEWS",@@ -173,7 +173,7 @@{:description"With each order of pies sent out, Piespace includes a place for customers to submit feedback and review their order.",:entity_type "entity/GenericTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "FEEDBACK",@@ -194,7 +194,7 @@{:description"Information on customer accounts registered with Piespace. Each account represents a new organization signing up for on-demand pies.",:entity_type "entity/UserTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "ACCOUNTS",@@ -215,7 +215,7 @@{:description"Piespace does some anonymous analytics tracking on how users interact with their platform. They’ve only had time to implement a few events, but you know how it is. Pies come first.",:entity_type "entity/EventTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "ANALYTIC_EVENTS",@@ -236,7 +236,7 @@{:description"Confirmed payments from Piespace’s customers. Most accounts pay for their pie subscription on a monthly basis.",:entity_type "entity/GenericTable",- :schema "PUBLIC",+ :schema nil,:database_require_filter nil,:show_in_getting_started false,:name "INVOICES",diff --git a/resources/sample-database.sqlite b/resources/sample-database.sqlitenew file mode 100644index 00000000000..fd26e781412Binary files /dev/null and b/resources/sample-database.sqlite differdiff --git a/src/metabase/app_db/custom_migrations.clj b/src/metabase/app_db/custom_migrations.cljindex 7028111cdff..2fdb9042497 100644--- a/src/metabase/app_db/custom_migrations.clj+++ b/src/metabase/app_db/custom_migrations.clj@@ -1330,6 +1330,121 @@(custom-migrations.util/with-temp-schedule! [scheduler](qs/delete-job scheduler (jobs/key "metabase.task.send-pulses.init-send-pulse-triggers.job"))))+(defn- card-ids-depending-on-database+ "Set of report_card ids that depend on `database-id` directly (their own query targets it) or transitively+ (a card built on such a card via source_card_id, to any depth)."+ [database-id]+ (loop [frontier (into #{} (map :id) (t2/query {:select [:id] :from [:report_card]+ :where [:= :database_id database-id]}))+ acc #{}]+ (if (empty? frontier)+ acc+ (let [acc' (into acc frontier)+ next* (into #{} (map :id) (t2/query {:select [:id] :from [:report_card]+ :where [:and [:in :source_card_id frontier]+ [:not [:in :id acc']]]}))]+ (recur next* acc')))))++(defn- collection-has-survivors?+ "True if `coll-id` still holds anything after sample content is gone: any content row filed into it, or any+ surviving child collection (whose path is exactly `child-location`). Checked bottom-up so a child that itself+ survived keeps its parent alive."+ [coll-id child-location]+ (boolean+ (or (seq (t2/query {:select [1] :from [:collection] :where [:= :location child-location] :limit 1}))+ (some (fn [table]+ (seq (t2/query {:select [1] :from [table] :where [:= :collection_id coll-id] :limit 1})))+ [:report_card :report_dashboard :native_query_snippet :pulse :timeline :document+ :metabase_table :transform]))))++(defn- remove-sqlite-sample-database-on-downgrade!+ "Delete the SQLite sample database and everything that depends on it - its tables/fields, every Card whose+ query targets it (directly or transitively through source_card_id), those cards' dashcards, and Dashboards+ left empty as a result - even when that content was user-created, because it cannot work without the database.++ The Example collections are then pruned, bottom-up: a sample collection is deleted only if nothing is left in+ it once the sample content is gone (no content rows, no surviving child collection); a collection a user kept+ their own content in is left intact. Permission records for the deleted collections go with them.++ Children are deleted explicitly, bottom-up, rather than relying on ON DELETE CASCADE: MySQL 9.7 resolves+ multi-level cascade fan-outs incompletely, leaving orphaned rows that later break ALTER TABLE statements which+ re-validate foreign keys."+ []+ (when-let [sample-db-id (:id (t2/query-one {:select [:id]+ :from [:metabase_database]+ :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))]+ (let [card-ids (card-ids-depending-on-database sample-db-id)+ field-ids-q {:select [:id]+ :from [:metabase_field]+ :where [:in :table_id {:select [:id]+ :from [:metabase_table]+ :where [:= :db_id sample-db-id]}]}+ affected-dashboard-ids (if (empty? card-ids)+ #{}+ (->> (t2/query {:select-distinct [:dashboard_id]+ :from [:report_dashboardcard]+ :where [:in :card_id card-ids]})+ (into #{} (map :dashboard_id))))]+ (when (seq card-ids)+ (t2/query {:delete-from :dashboardcard_series+ :where [:or+ [:in :card_id card-ids]+ [:in :dashboardcard_id {:select [:id]+ :from [:report_dashboardcard]+ :where [:in :card_id card-ids]}]]})+ (t2/query {:delete-from :report_dashboardcard :where [:in :card_id card-ids]})+ (t2/query {:delete-from :parameter_card+ :where [:or+ [:in :card_id card-ids]+ [:and [:= :parameterized_object_type "card"]+ [:in :parameterized_object_id card-ids]]]})+ (t2/query {:delete-from :report_card :where [:in :id card-ids]}))+ (t2/query {:delete-from :dimension :where [:in :field_id field-ids-q]})+ (t2/query {:delete-from :metabase_field+ :where [:in :table_id {:select [:id]+ :from [:metabase_table]+ :where [:= :db_id sample-db-id]}]})+ (t2/query {:delete-from :metabase_table :where [:= :db_id sample-db-id]})+ (t2/query {:delete-from :metabase_database :where [:= :id sample-db-id]})+ (when (seq affected-dashboard-ids)+ ;; A dashboard that depended on the sample DB is removed once no card-backed dashcard survives - even if+ ;; text/heading dashcards remain (the example dashboard is built entirely from sample cards plus headings).+ ;; A dashboard that still has a real card of its own (e.g. a user mixed in another card) is left alone.+ (let [non-empty (->> (t2/query {:select-distinct [:dashboard_id]+ :from [:report_dashboardcard]+ :where [:and [:in :dashboard_id affected-dashboard-ids]+ [:not= :card_id nil]]})+ (into #{} (map :dashboard_id)))+ empty-ids (remove non-empty affected-dashboard-ids)]+ (when (seq empty-ids)+ (t2/query {:delete-from :report_dashboardcard :where [:in :dashboard_id empty-ids]})+ (t2/query {:delete-from :dashboard_tab :where [:in :dashboard_id empty-ids]})+ (t2/query {:delete-from :parameter_card+ :where [:and [:= :parameterized_object_type "dashboard"]+ [:in :parameterized_object_id empty-ids]]})+ (t2/query {:delete-from :report_dashboard :where [:in :id empty-ids]}))))+ ;; Prune the Example collections deepest-first, so an emptied child is gone before its parent is judged.+ (doseq [{:keys [id location]} (->> (t2/query {:select [:id :location] :from [:collection] :where [:= :is_sample true]})+ (sort-by (comp count :location) >))]+ (when-not (collection-has-survivors? id (str location id "/"))+ (t2/query {:delete-from :permissions+ :where [:or+ [:= :collection_id id]+ [:in :object [(format "/collection/%d/" id)+ (format "/collection/%d/read/" id)]]]})+ (t2/query {:delete-from :collection :where [:= :id id]}))))))++;; The bundled sample database moved from H2 to SQLite. If an instance running a SQLite-sample+;; version is downgraded to an H2-sample version, the older code cannot use the SQLite sample+;; database (it has the wrong engine/details) and would leave it broken. On downgrade we therefore+;; remove the SQLite sample database and the content left dangling by it, and we do NOT restore an+;; H2 sample database. This mirrors the upgrade behavior in+;; metabase.sample-data.impl/replace-sample-database!. There is no forward migration: the upgrade+;; (H2 -> SQLite) replacement is handled at startup, not here.+(define-reversible-migration MigrateAwayFromSqliteSampleDatabaseOnDowngrade+ (log/info "No forward migration for MigrateAwayFromSqliteSampleDatabaseOnDowngrade")+ (remove-sqlite-sample-database-on-downgrade!))+;; when card display is area or bar,;; 1. set the display key to :stackable.stack_display value OR leave it the same;; 2. when series settings exist, remove the display key from each map in the series_settings listdiff --git a/src/metabase/core/core.clj b/src/metabase/core/core.cljindex d1c0feae94c..7d8b1b44f20 100644--- a/src/metabase/core/core.clj+++ b/src/metabase/core/core.clj@@ -214,12 +214,14 @@(setup/clear-token!))(init-status/set-progress! 0.7);; deal with our sample database as needed- (when (config/load-sample-content?)- (if new-install?- ;; add the sample database DB for fresh installs- (sample-data/extract-and-sync-sample-database!)- ;; otherwise update if appropriate- (sample-data/update-sample-database-if-needed!)))+ (if new-install?+ ;; add the sample database DB for fresh installs (only when sample content is enabled)+ (when (config/load-sample-content?)+ (sample-data/extract-and-sync-sample-database!))+ ;; On existing installs always reconcile: if the bundled engine changed (H2 <-> SQLite) the old+ ;; sample database must be cleaned up and replaced regardless of whether sample content is+ ;; currently enabled. Otherwise just refresh its connection details.+ (sample-data/update-sample-database-if-needed!))(init-status/set-progress! 0.8))(ensure-audit-db-installed!)(notification/seed-notification!)diff --git a/src/metabase/data_studio/api/table.clj b/src/metabase/data_studio/api/table.cljindex 5db8af40a6d..289557ccf8b 100644--- a/src/metabase/data_studio/api/table.clj+++ b/src/metabase/data_studio/api/table.clj@@ -153,7 +153,8 @@(let [database (t2/select-one :model/Database db-id)];; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the;; purposes of creating a new H2 database.- (if (binding [driver.settings/*allow-testing-h2-connections* true]+ (if (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](driver.u/can-connect-with-details? (:engine database) (:details database)))(doseq [table tables](log/info (u/format-color :green "Table '%s' is now visible. Resyncing." (:name table)))@@ -240,7 +241,8 @@db-ids (sort (set (map :db_id tables)))](doseq [database (t2/select :model/Database :id [:in db-ids])](try- (binding [driver.settings/*allow-testing-h2-connections* true]+ (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](driver.u/can-connect-with-details? (:engine database) (:details database) :throw-exceptions))nil(catch Throwable ediff --git a/src/metabase/driver/init.clj b/src/metabase/driver/init.cljindex c0318adf704..7e1a2ea768b 100644--- a/src/metabase/driver/init.clj+++ b/src/metabase/driver/init.clj@@ -13,4 +13,5 @@[metabase.driver.postgres][metabase.driver.settings];; for the `:sql-jdbc` implementation of [[metabase.driver/incorporate-ssh-tunnel-details]]- [metabase.driver.sql-jdbc.connection.ssh-tunnel]))+ [metabase.driver.sql-jdbc.connection.ssh-tunnel]+ [metabase.driver.sqlite]))diff --git a/src/metabase/driver/settings.clj b/src/metabase/driver/settings.cljindex b1f4ad0fafe..6a8ad505efd 100644--- a/src/metabase/driver/settings.clj+++ b/src/metabase/driver/settings.clj@@ -145,6 +145,12 @@(or (config/config-bool :mb-dangerous-unsafe-enable-testing-h2-connections-do-not-enable)false))+(def ^:dynamic *allow-testing-sqlite-connections*+ "Whether to allow testing new SQLite connections. Normally disabled on hosted Metabase, which effectively prevents+ users from creating new SQLite databases from the API. Internal flows that need to test connections to the bundled+ Sample Database (sync, schema refresh, fingerprinting, etc.) bind this to `true`."+ false)+(defn- -jdbc-data-warehouse-unreturned-connection-timeout-seconds [](or (setting/get-value-of-type :integer :jdbc-data-warehouse-unreturned-connection-timeout-seconds)(long (/ *query-timeout-ms* 1000))))diff --git a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj b/src/metabase/driver/sqlite.cljsimilarity index 92%rename from modules/drivers/sqlite/src/metabase/driver/sqlite.cljrename to src/metabase/driver/sqlite.cljindex b3dd68137a3..98839ef9e1a 100644--- a/modules/drivers/sqlite/src/metabase/driver/sqlite.clj+++ b/src/metabase/driver/sqlite.clj@@ -9,6 +9,8 @@[java-time.api :as t][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.common :as driver.common]+ [metabase.driver.settings :as driver.settings][metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]@@ -16,6 +18,7 @@[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp]+ [metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [tru]]@@ -30,6 +33,20 @@(driver/register! :sqlite, :parent :sql-jdbc)+(defmethod driver/display-name :sqlite+ [_driver]+ "SQLite")++(defmethod driver/connection-properties :sqlite+ [_driver]+ (into [] (mapcat u/one-or-many)+ [{:name "db"+ :display-name (tru "Filename")+ :placeholder "/path/to/toucan_sightings.sqlite"+ :required true}+ driver.common/advanced-options-start+ driver.common/default-advanced-options]))+;; SQLite does not support a lot of features, so do not show the options in the interface(doseq [[feature supported?] {:right-join false:full-join false@@ -65,8 +82,19 @@(or (str/includes? line "SQLite format 3")(str/includes? line "This file contains an SQLite"))))))+(defmethod driver/validate-db-details! :sqlite+ [_driver _details]+ ;; On hosted Metabase, SQLite is only valid for the bundled Sample Database. The internal flows that need to test+ ;; connections to that DB (sync, schema refresh, fingerprinting, etc.) wrap their calls in+ ;; `(binding [driver.settings/*allow-testing-sqlite-connections* true] ...)`. Mirrors the H2 pattern.+ (when (and (driver-api/is-hosted?)+ (not driver.settings/*allow-testing-sqlite-connections*))+ (throw (ex-info (tru "SQLite is not available as a data warehouse on Metabase Cloud.")+ {:status-code 400}))))+(defmethod driver/can-connect? :sqlite[driver details]+ (driver/validate-db-details! driver details)(if (confirm-file-is-sqlite (:db details))(sql-jdbc.conn/can-connect? driver details)false))@@ -75,13 +103,24 @@[_]:sunday)+(defn- read-only-open-mode+ "`open_mode` flag value that SQLite-JDBC uses for a read-only connection. Delegate to `SQLiteConfig` rather than+ hard-coding the bitmask so it tracks the driver's own definition."+ []+ (let [config (org.sqlite.SQLiteConfig.)]+ (.setReadOnly config true)+ (.getProperty (.toProperties config) "open_mode")))+(defmethod sql-jdbc.conn/connection-details->spec :sqlite- [_ {:keys [db]+ [_ {:keys [db read-only?]:or {db "sqlite.db"}:as details}](merge {:subprotocol "sqlite":subname db}- (dissoc details :db)+ (dissoc details :db :read-only?)+ ;; SQLite-JDBC can't flip a live connection to read-only, so set it at open time via `open_mode`.+ (when read-only?+ {:open_mode (read-only-open-mode)});; disallow "FDW" (connecting to other SQLite databases on the local filesystem) -- see https://github.com/metabase/metaboat/issues/152{:limit_attached 0}))diff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.cljindex 69da8c3ee0e..7f6691b4c2f 100644--- a/src/metabase/driver/util.clj+++ b/src/metabase/driver/util.clj@@ -283,17 +283,11 @@f (if *memoize-supports?* memoized-features* features*)](f driver database)))-(mu/defn- supported-in-environment?- "Returns true if a driver is supported in the the current metabase environment. As implemented this just disallows the- sqlite driver on hosted metabase because hosted metabase does not support uploading a SQLite file for use."- [driver :- :keyword]- (or (not (premium-features/is-hosted?))- (not= :sqlite (keyword driver))))-(defn available-drivers"Return a set of all currently available drivers."[]- (into #{} (filter #(and (driver/available? %) (supported-in-environment? %)))+ (into #{}+ (filter driver/available?)(descendants driver/hierarchy :metabase.driver/driver)))(mu/defn semantic-version-gte :- :boolean@@ -601,27 +595,41 @@"official""community"))+(defn- creatable?+ "Whether users may select this driver to create a new data warehouse connection, given `context` -- a map describing+ the environment, currently `{:hosted? <boolean>}`. Defaults to true. A driver registered only to power an+ internal/bundled database -- e.g. SQLite, which backs the bundled Sample Database but is not offered to Cloud users+ as a warehouse -- returns false in the contexts where it is not user-creatable, so it is omitted from the+ add-database engine list there. Affects only the engine list shown to users; the driver stays registered and+ `available?` for internal use."+ [driver {:keys [hosted?]}]+ (not (and+ (= driver :sqlite)+ hosted?)))+(defn available-drivers-info"Return info about all currently available drivers, including their connection properties fields and supportedfeatures. The output of `driver/connection-properties` is passed through `connection-props-server->client` beforebeing returned, to handle any transformation between the server side and client side representation."[]- (persistent!- (reduce (fn [acc driver]- (if-some [props (try- (->> (driver/connection-properties driver)- (connection-props-server->client driver))- (catch Throwable e- (log/errorf e "Unable to determine connection properties for driver %s" driver)))]- ;; TODO - maybe we should rename `details-fields` -> `connection-properties` on the FE as well?- (assoc! acc driver {:source {:type (driver-source (name driver))- :contact (driver/contact-info driver)}- :details-fields props- :driver-name (driver/display-name driver)- :superseded-by (driver/superseded-by driver)- :extra-info (driver/extra-info driver)})- acc))- (transient {}) (available-drivers))))+ (let [context {:hosted? (premium-features/is-hosted?)}]+ (persistent!+ (reduce (fn [acc driver]+ (if-some [props (try+ (->> (driver/connection-properties driver)+ (connection-props-server->client driver))+ (catch Throwable e+ (log/errorf e "Unable to determine connection properties for driver %s" driver)))]+ ;; TODO - maybe we should rename `details-fields` -> `connection-properties` on the FE as well?+ (assoc! acc driver {:source {:type (driver-source (name driver))+ :contact (driver/contact-info driver)}+ :details-fields props+ :driver-name (driver/display-name driver)+ :superseded-by (driver/superseded-by driver)+ :creatable? (creatable? driver context)+ :extra-info (driver/extra-info driver)})+ acc))+ (transient {}) (available-drivers)))));;; +----------------------------------------------------------------------------------------------------------------+;;; | TLS Helpers |diff --git a/src/metabase/sample_data/example_content.clj b/src/metabase/sample_data/example_content.cljnew file mode 100644index 00000000000..06e1fc65cfa--- /dev/null+++ b/src/metabase/sample_data/example_content.clj@@ -0,0 +1,285 @@+(ns metabase.sample-data.example-content+ "Recreate the bundled Example collection (the content shipped in `sample-content.edn`) on top of an+ already-synced Sample Database.++ On a fresh install `metabase.app-db.custom-migrations/CreateSampleContentV2` inserts the EDN rows+ verbatim into empty tables, so auto-increment reproduces the EDN ids exactly and every embedded+ reference still lines up. On upgrade the tables are not empty, so we insert the same rows but remap+ each id: simple FK columns by lookup, and the Database/Table/Field/Card ids embedded in card and+ dashcard query blobs via the serdes export walkers bound to numeric id maps."+ (:require+ [clojure.edn :as edn]+ [clojure.java.io :as io]+ [clojure.string :as str]+ [clojure.walk :as walk]+ [medley.core :as m]+ [metabase.models.interface :as mi]+ [metabase.models.serialization :as serdes]+ [metabase.settings.core :as setting]+ [metabase.util.date-2 :as u.date]+ [metabase.util.json :as json]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(def ^:private sample-content-resource "sample-content.edn")++(defn- load-sample-content []+ (with-open [r (io/reader (io/resource sample-content-resource))]+ (edn/read {:readers {'t u.date/parse}} (java.io.PushbackReader. r))))++(defn- now-for-temporals [row]+ (update-vals row (fn [v] (if (instance? java.time.temporal.Temporal v) :%now v))))++(defn- prep-row+ "Strip the EDN id (let the app DB assign one) and replace temporal literals with `:%now`."+ [row]+ (-> row now-for-temporals (dissoc :id)))++(defn- insert-rows!+ "Insert `rows` (each carrying its original EDN `:id` and `:entity_id`) into `table` after prepping,+ and return a map of `edn-id -> new id` so other rows' numeric FK references can be remapped. The new+ id is recovered by matching the unique `:entity_id` back to the inserted row."+ [table rows]+ (when (seq rows)+ (t2/query {:insert-into table :values (mapv prep-row rows)}))+ (let [eid->new (into {}+ (map (juxt :entity_id :id))+ (t2/query {:select [:id :entity_id] :from table+ :where [:in :entity_id (mapv :entity_id rows)]}))]+ (into {} (map (fn [r] [(:id r) (eid->new (:entity_id r))])) rows)))++;;; --------------------------------------------- id remapping ----------------------------------------------++(defn- model->map-key [model]+ (case (name model)+ "Card" :cards+ "Dashboard" :dashboards+ "Collection" :collections+ nil))++(defn- remap-result-metadata [maps cols]+ (mapv (fn [m]+ (-> m+ (m/update-existing :table_id #(get (:tables maps) % %))+ (m/update-existing :id #(if (number? %) (get (:fields maps) % %) %))+ (m/update-existing :field_ref serdes/export-mbql)+ (m/update-existing :fk_target_field_id #(if (number? %) (get (:fields maps) % %) %))))+ cols))++(defn- remap-source-card-refs+ "`export-mbql` only remaps a numeric `:source-table` (a table id); a `\"card__N\"` `:source-table` - a card+ that builds on another card - is left carrying the EDN card id. Rewrite each through the card id map so it+ points at the recreated card instead of whatever row happens to occupy the EDN id on an upgraded app db."+ [card-map query]+ (walk/postwalk+ (fn [x]+ (if-let [[_ n] (and (map? x)+ (string? (:source-table x))+ (re-matches #"card__(\d+)" (:source-table x)))]+ (let [old-id (parse-long n)]+ (assoc x :source-table (str "card__" (get card-map old-id old-id))))+ x))+ query))++(defn- remap-blob+ "Remap the entity ids embedded in a serialized JSON `blob` string of the given `kind`, returning a+ JSON string. Reuses the serdes export walkers with the FK resolvers bound to return new numeric ids+ (rather than portable references)."+ [maps kind blob]+ (when (some? blob)+ (binding [serdes/*export-database-fk* (fn [_id] (:db maps))+ serdes/*export-field-fk* (fn [id] (get (:fields maps) id id))+ serdes/*export-table-fk* (fn [id] (get (:tables maps) id id))+ serdes/*export-fk* (fn [id model]+ (let [k (model->map-key model)]+ (if k (get (get maps k) id id) id)))]+ (json/encode+ (case kind+ :mbql (->> (json/decode+kw blob) serdes/export-mbql (remap-source-card-refs (:cards maps)))+ ;; Remap each parameter's refs in place, preserving the designed order. (export-parameters would reorder+ ;; them by :id - a serialization-stability device we must not inherit, or the filter order is scrambled.)+ :parameters (mapv serdes/export-mbql (json/decode+kw blob))+ :param-maps (serdes/export-parameter-mappings (json/decode+kw blob))+ ;; column_settings keys must stay strings, so decode without keywordizing (as the model does).+ :viz (serdes/export-visualization-settings (mi/normalize-visualization-settings (json/decode blob)))+ :result-md (remap-result-metadata maps (json/decode+kw blob)))))))++;;; --------------------------------------------- insertion ----------------------------------------------++(defn- field-id-maps+ "Build `{:db new-db-id :tables {edn-table-id new-table-id} :fields {edn-field-id new-field-id}}` by+ matching the EDN table/field names against the freshly-synced Sample Database."+ [content new-db-id]+ (let [edn-tables (:metabase_table content)+ edn-fields (:metabase_field content)+ tname->new (t2/select-fn->pk :name :model/Table :db_id new-db-id)+ table-id-map (into {} (keep (fn [{:keys [id name]}]+ (when-let [new-id (tname->new name)]+ [id new-id])))+ edn-tables)+ ;; [new-table-id field-name] -> new-field-id+ new-fields (t2/select :model/Field :table_id [:in (vals table-id-map)])+ nf-index (into {} (map (fn [f] [[(:table_id f) (:name f)] (:id f)])) new-fields)+ field-id-map (into {} (keep (fn [{:keys [id table_id name]}]+ (when-let [new-tid (table-id-map table_id)]+ (when-let [new-fid (nf-index [new-tid name])]+ [id new-fid])))+ edn-fields))]+ {:db new-db-id :tables table-id-map :fields field-id-map}))++(defn- remap-collection-location+ "Remap the parent ids embedded in a collection `location` path (e.g. \"/2/3/\") through `old->new`, one path+ segment at a time. Remapping per segment (rather than successive string replacement over the id map) avoids a+ newly-assigned id being re-substituted as if it were a still-pending old id."+ [old->new location]+ (if (= "/" location)+ location+ (str "/"+ (->> (str/split location #"/")+ (remove str/blank?)+ (map (fn [segment]+ (if-let [old-id (parse-long segment)]+ (str (get old->new old-id old-id))+ segment)))+ (str/join "/"))+ "/")))++(defn- insert-collections! [content]+ ;; Reuse the existing Example collections (matched by entity id) so user content filed into them survives an+ ;; engine swap; only create one when it's missing. Roots before children so a new child's location path can be+ ;; remapped against ids already assigned.+ (loop [pending (sort-by (comp count :location) (:collection content))+ coll-map {}]+ (if-let [row (first pending)]+ (let [existing (:id (t2/query-one {:select [:id] :from :collection :where [:= :entity_id (:entity_id row)]}))+ new-id (or existing+ (let [prepared (-> row prep-row (update :location #(remap-collection-location coll-map %)))]+ (t2/query {:insert-into :collection :values [prepared]})+ (:id (t2/query-one {:select [:id] :from :collection :where [:= :entity_id (:entity_id row)]}))))]+ (recur (rest pending) (assoc coll-map (:id row) new-id)))+ coll-map)))++(defn- delete-prior-sample-content!+ "Drop the previously-bundled sample cards and dashboards, matched by their stable EDN entity ids, so reinserting+ them can't collide and stale ones don't linger. User content never carries these entity ids, so it is untouched;+ the Example collections are reused, not deleted. Dashboards are removed before cards because deleting a dashboard+ cascades its dashcards."+ [content]+ (doseq [table [:report_dashboard :report_card]]+ (let [eids (into [] (keep :entity_id) (get content table))]+ (when (seq eids)+ (t2/query {:delete-from table :where [:in :entity_id eids]})))))++(defn- insert-cards! [content maps]+ ;; Phase 1: insert with simple FKs remapped and blobs left as-is (no FK enforcement on JSON), so all+ ;; card ids exist before we rewrite inter-card references in phase 2.+ (let [coll-map (:collections maps)+ rows (for [card (:report_card content)]+ (-> card+ (assoc :database_id (:db maps))+ (assoc :collection_id (some-> (:collection_id card) coll-map))+ (assoc :table_id (some-> (:table_id card) (->> (get (:tables maps)))))+ (assoc :source_card_id nil)))+ card-map (insert-rows! :report_card (vec rows))+ maps (assoc maps :cards card-map)]+ ;; Phase 2: rewrite query blobs + inter-card refs now that every card id exists.+ (doseq [card (:report_card content)]+ (t2/query {:update :report_card+ :set {:dataset_query (remap-blob maps :mbql (:dataset_query card))+ :result_metadata (remap-blob maps :result-md (:result_metadata card))+ :parameters (remap-blob maps :parameters (:parameters card))+ :parameter_mappings (remap-blob maps :param-maps (:parameter_mappings card))+ :visualization_settings (remap-blob maps :viz (:visualization_settings card))+ :source_card_id (some-> (:source_card_id card) card-map)}+ :where [:= :id (card-map (:id card))]}))+ maps))++(defn- insert-dashboards! [content maps]+ (let [coll-map (:collections maps)+ rows (mapv (fn [d] (assoc d :collection_id (some-> (:collection_id d) coll-map)))+ (:report_dashboard content))+ dash-map (insert-rows! :report_dashboard rows)+ maps (assoc maps :dashboards dash-map)]+ (doseq [d (:report_dashboard content)]+ (t2/query {:update :report_dashboard+ :set {:parameters (remap-blob maps :parameters (:parameters d))}+ :where [:= :id (dash-map (:id d))]}))+ maps))++(defn- insert-dashboard-tabs! [content maps]+ (let [dash-map (:dashboards maps)+ rows (mapv (fn [t] (assoc t :dashboard_id (dash-map (:dashboard_id t))))+ (:dashboard_tab content))]+ (assoc maps :tabs (insert-rows! :dashboard_tab rows))))++(defn- insert-dashcards! [content maps]+ (let [{:keys [dashboards cards tabs]} maps+ rows (mapv (fn [dc]+ (-> (prep-row dc)+ (assoc :dashboard_id (dashboards (:dashboard_id dc)))+ (assoc :card_id (some-> (:card_id dc) cards))+ (assoc :dashboard_tab_id (some-> (:dashboard_tab_id dc) tabs))+ (assoc :parameter_mappings (remap-blob maps :param-maps (:parameter_mappings dc)))+ (assoc :visualization_settings (remap-blob maps :viz (:visualization_settings dc)))))+ (:report_dashboardcard content))]+ (when (seq rows)+ (t2/query {:insert-into :report_dashboardcard :values rows}))+ maps))++(defn- insert-dimensions! [content maps]+ (let [fields (:fields maps)+ rows (mapv (fn [dim]+ (-> (prep-row dim)+ (assoc :field_id (fields (:field_id dim)))+ (m/update-existing :human_readable_field_id #(some->> % (get fields)))))+ (:dimension content))]+ (when (seq rows)+ (t2/query {:insert-into :dimension :values rows}))))++(defn- grant-example-collection-perms! [content maps]+ (when-let [example-collection-id (some->> (:collection content)+ (filter :is_sample)+ first :id+ (get (:collections maps)))]+ (when-let [group-id (:id (t2/query-one {:select [:id] :from :permissions_group :where [:= :name "All Users"]}))]+ ;; A reused Example collection already carries this grant; only insert when it's missing.+ (when-not (seq (t2/query {:select [1] :from :permissions+ :where [:and [:= :collection_id example-collection-id]+ [:= :group_id group-id]+ [:= :perm_type "perms/collection-access"]]+ :limit 1}))+ (t2/query {:insert-into :permissions+ :values [{:object (format "/collection/%s/" example-collection-id)+ :group_id group-id+ :perm_type "perms/collection-access"+ :perm_value "read-and-write"+ :collection_id example-collection-id}]})))))++(defn- set-example-dashboard-id! [content maps]+ (when-let [dash (first (:report_dashboard content))]+ (when-let [new-id (get (:dashboards maps) (:id dash))]+ (setting/set-value-of-type! :integer :example-dashboard-id new-id))))++(defn recreate-example-content!+ "Recreate the bundled Example collection on the freshly-synced Sample Database `new-db-id`. Mirrors+ the fresh-install seeding, remapping every id reference to point at the new database's tables and+ fields. Best-effort: logs and swallows errors so a content-seeding failure can't break startup."+ [new-db-id]+ (try+ (t2/with-transaction [_conn]+ (let [content (load-sample-content)+ _ (delete-prior-sample-content! content)+ maps (field-id-maps content new-db-id)+ maps (assoc maps :collections (insert-collections! content))+ maps (insert-cards! content maps)+ maps (insert-dashboards! content maps)+ maps (insert-dashboard-tabs! content maps)+ maps (insert-dashcards! content maps)]+ (insert-dimensions! content maps)+ (grant-example-collection-perms! content maps)+ (set-example-dashboard-id! content maps)))+ (log/info "Recreated Sample Database example content")+ (catch Throwable e+ (log/error e "Failed to recreate Sample Database example content"))))diff --git a/src/metabase/sample_data/impl.clj b/src/metabase/sample_data/impl.cljindex 779d5451937..e4bb0ed4417 100644--- a/src/metabase/sample_data/impl.clj+++ b/src/metabase/sample_data/impl.clj@@ -3,7 +3,9 @@(:require[clojure.java.io :as io][clojure.string :as str]+ [metabase.config.core :as config][metabase.plugins.core :as plugins]+ [metabase.sample-data.example-content :as example-content][metabase.sync.core :as sync][metabase.util.files :as u.files][metabase.util.i18n :refer [trs]]@@ -20,7 +22,15 @@identify or validate the sample DB (e.g. workspace-manager config import).""Sample Database")-(def ^:private ^String sample-database-filename "sample-database.db.mv.db")+(def ^:private ^String sqlite-sample-database-filename "sample-database.sqlite")++(def ^:private ^String h2-sample-database-filename "sample-database.db.mv.db")++(defn- sample-database-filename+ [engine]+ (case engine+ :h2 h2-sample-database-filename+ :sqlite sqlite-sample-database-filename))(defn- sample-db-dir-from-env"Change the folder from which we load the sample database, used locally to avoid E2E to use the same file used for local development"@@ -31,51 +41,54 @@;; Reuse the plugins directory for the destination to extract the sample database because it's pretty much guaranteed;; to exist and be writable.(defn- target-path- []+ [engine](let [base-dir (or (sample-db-dir-from-env)(plugins/plugins-dir))]- (u.files/append-to-path base-dir sample-database-filename)))--(defn- process-sample-db-path- [base-path]- (-> base-path- (str/replace #"\.mv\.db$" "") ; strip the .mv.db suffix from the path- codec/url-decode ; for some reason the path can get URL-encoded so we decode it here- (str ";USER=GUEST;PASSWORD=guest"))) ; specify the GUEST user account created for the DB--(defn- jar-db-details- [^URL resource]- (-> (.getPath resource)- (str/replace #"^file:" "zip:") ; to connect to an H2 DB inside a JAR just replace file: with zip: (this doesn't- ; do anything when running from the Clojure CLI, which has no `file:` prefix)- process-sample-db-path))+ (u.files/append-to-path base-dir (sample-database-filename engine))))(defn- extract-sample-database!- []- (u.files/with-open-path-to-resource [sample-db-path sample-database-filename]- (let [dest-path (target-path)]+ "Copy the bundled sample database file out to the plugins dir and return its destination path."+ [engine]+ (u.files/with-open-path-to-resource [sample-db-path (sample-database-filename engine)]+ (let [dest-path (target-path engine)](u.files/copy-file! sample-db-path dest-path)- (-> (str "file:" dest-path)- process-sample-db-path))))+ (str dest-path))))++(defn- sample-database-details+ "Build the `:details` map for the sample database extracted to `dest-path`, per engine."+ [engine dest-path]+ (case engine+ ;; SQLite `:details` only need a filesystem path. URL-decode in case it got URL-encoded from a JAR `URL`.+ ;; The bundled file is read-only.+ :sqlite {:db (codec/url-decode dest-path), :read-only? true}+ ;; H2 connects to the file by path (sans the `.mv.db` suffix).+ :h2 {:db (-> (str "file:" dest-path)+ (str/replace #"\.mv\.db$" "")+ codec/url-decode+ (str ";USER=GUEST;PASSWORD=guest"))}))(defn- try-to-extract-sample-database!- "Tries to extract the sample database out of the JAR (for performance) and then returns a db-details map- containing a path to the copied database."- []- (let [resource (io/resource sample-database-filename)]+ "Extracts the sample database out of the JAR to the plugins dir and returns a db-details map. SQLite-JDBC+ requires a real file on disk, so the plugins-dir-must-be-writable assumption is load-bearing."+ [engine]+ (let [filename (sample-database-filename engine)+ ^URL resource (io/resource filename)](when-not resource- (throw (Exception. (trs "Sample database DB file ''{0}'' cannot be found."- sample-database-filename))))- {:db- (if-not (:temp (plugins/plugins-dir-info))- (extract-sample-database!)- (do- ;; If the plugins directory is a temp directory, fall back to reading the DB directly from the JAR until a- ;; working plugins directory is available. (We want to ensure the sample DB is in a stable location.)- (log/warn (str "Sample database could not be extracted to the plugins directory,"- "which may result in slow startup times. "- "Please set MB_PLUGINS_DIR to a writable directory and restart Metabase."))- (jar-db-details resource)))}))+ (throw (ex-info (trs "Sample database file ''{0}'' cannot be found." filename)+ {:filename filename})))+ (when (:temp (plugins/plugins-dir-info))+ (log/warn (str "Plugins directory is a temp directory; the sample database will be re-extracted on every startup. "+ "Set MB_PLUGINS_DIR to a writable directory to make this stable.")))+ (sample-database-details engine (extract-sample-database! engine))))++(defn- sample-database-engine+ "Engine of the bundled sample database. Defaults to `:sqlite` (what we ship). E2E tests set+ `MB_SAMPLE_DATABASE_ENGINE=h2` so the sample database is the H2 one the suite was written against,+ deferring the test migration to SQLite without changing what we ship."+ []+ (if (= "h2" (System/getenv "MB_SAMPLE_DATABASE_ENGINE"))+ :h2+ :sqlite))(defn extract-and-sync-sample-database!"Adds the sample database as a Metabase DB if it doesn't already exist. If it does exist in the app DB,@@ -83,13 +96,14 @@[](try(log/info "Loading sample database")- (let [details (try-to-extract-sample-database!)+ (let [engine (sample-database-engine)+ details (try-to-extract-sample-database! engine)db (if (t2/exists? :model/Database :is_sample true)(t2/select-one :model/Database (first (t2/update-returning-pks! :model/Database :is_sample true {:details details})))(first (t2/insert-returning-instances! :model/Database:name sample-database-name:details details- :engine :h2+ :engine engine:is_sample true)))](log/debug "Syncing Sample Database...")(sync/sync-database! db))@@ -97,13 +111,79 @@(catch Throwable e(log/error e "Failed to load sample database"))))+(defn- sample-database-dashboard-ids+ "IDs of Dashboards holding at least one card backed by `database-id`. Captured before the sample+ database is deleted so the dashboards it empties out can be pruned afterward."+ [database-id]+ (t2/select-fn-set :dashboard_id :model/DashboardCard+ :card_id [:in {:select [:id]+ :from [(t2/table-name :model/Card)]+ :where [:= :database_id database-id]}]))++(defn- delete-emptied-dashboards!+ "Delete whichever of `dashboard-ids` no longer have any dashcards. A sample dashboard whose cards+ were all removed with the sample database is deleted; a dashboard that still has cards (e.g. mixes+ in cards from another database) is left alone."+ [dashboard-ids]+ (when (seq dashboard-ids)+ (let [non-empty (t2/select-fn-set :dashboard_id :model/DashboardCard :dashboard_id [:in dashboard-ids])+ empty-ids (remove (or non-empty #{}) dashboard-ids)]+ (when (seq empty-ids)+ (t2/delete! :model/Dashboard :id [:in empty-ids])))))++(defn- replace-sample-database!+ "The bundled sample database's engine changed (e.g. H2 -> SQLite on upgrade). The old sample+ Database's tables, fields, and connection details are incompatible with the new engine, so rather+ than remapping content we drop the old sample Database wholesale and extract + sync the new one.+ Deleting the Database cascades to its Cards and those cards' dashcards; Dashboards left empty as a+ result are deleted too.++ The cleanup runs unconditionally (an incompatible sample database should never linger), but the new+ Sample Database and its Example collection are only recreated when sample content is enabled - and+ the Example collection is skipped under test endpoints, matching fresh-install seeding.++ The Example collection itself is left in place: [[example-content/recreate-example-content!]] reuses it+ (matching by entity id) and replaces only the bundled cards/dashboards, so any content a user filed into+ the Example collection survives the engine swap."+ [engine old-sample-db]+ (log/infof "Bundled sample database engine changed from %s to %s; replacing the sample database"+ (:engine old-sample-db) engine)+ ;; Extract the bundled DB file (a multi-MB copy out of the JAR) before opening the transaction, so the file IO+ ;; doesn't hold the app-DB transaction open.+ (let [new-db-details (when (config/load-sample-content?)+ (try-to-extract-sample-database! engine))+ new-db (t2/with-transaction [_conn]+ (let [dashboard-ids (sample-database-dashboard-ids (:id old-sample-db))]+ (t2/delete! :model/Database (:id old-sample-db))+ (delete-emptied-dashboards! dashboard-ids))+ (when new-db-details+ (first (t2/insert-returning-instances! :model/Database+ :name sample-database-name+ :details new-db-details+ :engine engine+ :is_sample true))))]+ (when new-db+ (when (and (try+ (sync/sync-database! new-db)+ true+ (catch Throwable e+ (log/error e "Failed to sync the replacement sample database")+ false))+ (not (config/config-bool :mb-enable-test-endpoints)))+ (example-content/recreate-example-content! (:id new-db))))))+(defn update-sample-database-if-needed!- "Update the path to the sample database DB if it exists in case the JAR has moved."+ "Reconcile the existing sample database with the bundled one. When the bundled engine changed+ (H2 <-> SQLite) the old sample database is replaced (see [[replace-sample-database!]]); otherwise+ we just refresh its connection details in case the JAR has moved."([](update-sample-database-if-needed! (t2/select-one :model/Database :is_sample true)))([sample-db](when sample-db- (let [intended (try-to-extract-sample-database!)]- (when (not= (:details sample-db) intended)- (t2/update! :model/Database (:id sample-db) {:details intended}))))))+ (let [engine (sample-database-engine)]+ (if (not= (:engine sample-db) engine)+ (replace-sample-database! engine sample-db)+ (let [intended (try-to-extract-sample-database! engine)]+ (when (not= (:details sample-db) intended)+ (t2/update! :model/Database (:id sample-db) {:details intended}))))))))diff --git a/src/metabase/sync/sync.clj b/src/metabase/sync/sync.cljindex a45761ade57..8b0c1a2238a 100644--- a/src/metabase/sync/sync.clj+++ b/src/metabase/sync/sync.clj@@ -95,7 +95,8 @@database (table/database table)];; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the;; purposes of creating a new H2 database.- (if (binding [driver.settings/*allow-testing-h2-connections* true]+ (if (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](driver.u/can-connect-with-details? (:engine database) (:details database)))(sync-util/with-error-handling (format "Error refingerprinting field %s"(sync-util/name-for-logging field))diff --git a/src/metabase/sync/task/sync_databases.clj b/src/metabase/sync/task/sync_databases.cljindex fcd2adb8570..dc4d5c48d64 100644--- a/src/metabase/sync/task/sync_databases.clj+++ b/src/metabase/sync/task/sync_databases.clj@@ -83,7 +83,8 @@(if-let [ex (try;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the;; purposes of creating a new H2 database.- (binding [driver.settings/*allow-testing-h2-connections* true]+ (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](driver.u/can-connect-with-details? (:engine database) (:details database) :throw-exceptions))nil(catch Throwable ediff --git a/src/metabase/warehouse_schema_rest/api/table.clj b/src/metabase/warehouse_schema_rest/api/table.cljindex e23b206201f..e139afd5ff6 100644--- a/src/metabase/warehouse_schema_rest/api/table.clj+++ b/src/metabase/warehouse_schema_rest/api/table.clj@@ -218,7 +218,8 @@(let [database (t2/select-one :model/Database db-id)];; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the;; purposes of creating a new H2 database.- (if (binding [driver.settings/*allow-testing-h2-connections* true]+ (if (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](driver.u/can-connect-with-details? (:engine database) (:details database)))(doseq [table tables](log/info (u/format-color :green "Table '%s' is now visible. Resyncing." (:name table)))@@ -531,7 +532,8 @@;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the;; purposes of creating a new H2 database.(if-let [ex (try- (binding [driver.settings/*allow-testing-h2-connections* true]+ (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](driver.u/can-connect-with-details? (:engine database) (:details database) :throw-exceptions))nil(catch Throwable ediff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.cljindex 320b1d02a06..1bf7dabd0c4 100644--- a/src/metabase/warehouses/models/database.clj+++ b/src/metabase/warehouses/models/database.clj@@ -691,7 +691,9 @@(t2/reducible-select (keyword "model" model-name){:where (cond-> [:and(or where true)- [:= :router_database_id nil]]+ [:= :router_database_id nil]+ ;; never export the sample database, regardless of its driver+ [:not= :is_sample true]](not *include-h2-in-extract?*)(conj [:not= :engine "h2"]))}))diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex da647b8fe7b..1b9db4625e2 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -1111,6 +1111,12 @@(throw (ex-info (tru "admin-connection must not be set in details"){:status-code 400})))(let [existing-database (api/write-check (t2/select-one :model/Database :id id))+ ;; e2e tests run against the H2 sample database and need to toggle its settings (actions,+ ;; table editing), so the guard is lifted when test endpoints are enabled+ _ (when (and (:is_sample existing-database)+ (not (config/config-bool :mb-enable-test-endpoints)))+ (throw (ex-info (tru "The sample database cannot be edited.")+ {:status-code 400})))_ (when write_data_details(validate-write-data-details! existing-database write_data_details))_ (when admin_details@@ -1266,7 +1272,8 @@(if-let [ex (try;; it's okay to allow testing H2 connections during sync. We only want to disallow you from testing them for the;; purposes of creating a new H2 database.- (binding [driver.settings/*allow-testing-h2-connections* true]+ (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](driver.u/can-connect-with-details? (:engine db) (driver.conn/default-details db) :throw-exceptions))nil(catch Throwable e@@ -1630,7 +1637,8 @@connection-details (driver.conn/details-for-exact-type database connection-type)](api/check-400 connection-details (tru "No {0} connection configured for this database" (name connection-type)));; we only want to prevent creating new H2 databases. Testing the existing database is fine.- (binding [driver.settings/*allow-testing-h2-connections* true]+ (binding [driver.settings/*allow-testing-h2-connections* true+ driver.settings/*allow-testing-sqlite-connections* true](if-let [err-map (warehouses/test-database-connection engine connection-details)](merge err-map {:status "error"}){:status "ok"}))))diff --git a/test/metabase/app_db/custom_migrations_test.clj b/test/metabase/app_db/custom_migrations_test.cljindex 56062d4608a..33d7130f95f 100644--- a/test/metabase/app_db/custom_migrations_test.clj+++ b/test/metabase/app_db/custom_migrations_test.clj@@ -2938,3 +2938,200 @@(is (= "metabase-transform" (:data_source provisional)))(is (= "computed" (:data_authority provisional)))(is (= "New Target Table" (:display_name provisional)))))))))++(deftest ^:mb/old-migrations-test migrate-away-from-sqlite-sample-database-on-downgrade-rollback-test+ (testing "Downgrading past the changeset (the real rollback path - the target version has none of the new+ sample database code) removes the SQLite sample database and the whole Example collection tree that+ the CreateSampleContent migration installed"+ ;; Start early enough that CreateSampleContent runs under our binding and installs the real SQLite Sample+ ;; Database and Example collection tree - the exact state a SQLite-sample version leaves to be torn down.+ (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]+ (binding [custom-migrations/*create-sample-content* true]+ (migrate!))+ (let [sample-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]+ :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))+ sample-coll-ids (mapv :id (t2/query {:select [:id] :from [:collection] :where [:= :is_sample true]}))]+ (testing "precondition: the SQLite sample database and its Example collections exist after upgrade"+ (is (some? sample-db-id))+ (is (seq sample-coll-ids)))+ (migrate! :down 62)+ (testing "the SQLite sample database is gone"+ (is (not (t2/exists? (t2/table-name :model/Database) :id sample-db-id))))+ (testing "the entire outgoing Example collection tree is deleted - the cascade is complete"+ (is (zero? (t2/count (t2/table-name :model/Collection) :id [:in sample-coll-ids]))))))))++(deftest migrate-away-from-sqlite-sample-database-on-downgrade-test+ (testing "Downgrade removes the SQLite sample database and the content it leaves empty, keeping everything else"+ (mt/with-temp+ [:model/Database sample {:engine :sqlite, :is_sample true, :details {:db "mem:sample"}}+ :model/Database other {:engine :h2, :details {:db "mem:other"}}+ :model/Card other-card {:database_id (:id other)}+ :model/Dashboard sample-dash {}+ :model/Dashboard mixed-dash {}+ :model/Collection examples {:name "Examples", :is_sample true}+ :model/Collection ecommerce {:name "E-commerce", :is_sample true, :location (str "/" (:id examples) "/")}+ :model/Collection keep-coll {:name "Keep me"}]+ ;; Mirror the bundled sample content's fan-out: 8 tables x 7 fields, 39 cards, plus dashcards,+ ;; series, and tabs. MySQL 9.7 resolves a multi-level ON DELETE CASCADE of this shape+ ;; incompletely (it deletes the tables but orphans most cards and all fields), so the cleanup+ ;; must not rely on DB-level cascade. This fixture makes that regression visible.+ (let [table-ids (vec (t2/insert-returning-pks! :model/Table+ (for [i (range 8)]+ {:db_id (:id sample)+ :name (str "sample_table_" i)+ :active true})))+ _ (t2/insert! :model/Field+ (for [tid table-ids+ i (range 7)]+ {:table_id tid+ :name (str "field_" i)+ :base_type :type/Text+ :database_type "TEXT"+ :position i}))+ card-ids (vec (t2/insert-returning-pks! :model/Card+ (for [i (range 39)]+ {:name (str "sample card " i)+ :display "table"+ :dataset_query {}+ :visualization_settings {}+ :creator_id (mt/user->id :rasta)+ :database_id (:id sample)+ :table_id (nth table-ids (mod i (count table-ids)))})))+ [dc1 _dc2 dc3] (t2/insert-returning-pks! :model/DashboardCard+ [{:dashboard_id (:id sample-dash) :card_id (first card-ids)+ :size_x 4 :size_y 4 :row 0 :col 0}+ {:dashboard_id (:id mixed-dash) :card_id (second card-ids)+ :size_x 4 :size_y 4 :row 0 :col 0}+ {:dashboard_id (:id mixed-dash) :card_id (:id other-card)+ :size_x 4 :size_y 4 :row 0 :col 4}])+ _ (t2/insert! :model/DashboardCardSeries {:dashboardcard_id dc1+ :card_id (nth card-ids 2)+ :position 0})+ _ (t2/insert! :model/DashboardTab {:dashboard_id (:id sample-dash)+ :name "Tab 1"+ :position 0})+ ;; parameter_card rows orphan in three ways; a fourth must survive.+ [pc-src+ pc-card-owner+ pc-dash-owner+ pc-keep] (t2/insert-returning-pks!+ :model/ParameterCard+ [;; value source is a sample card -> delete+ {:card_id (first card-ids) :parameterized_object_type "dashboard"+ :parameterized_object_id (:id mixed-dash) :parameter_id "src"}+ ;; parameterized object is a sample card -> delete+ {:card_id (:id other-card) :parameterized_object_type "card"+ :parameterized_object_id (second card-ids) :parameter_id "card-owner"}+ ;; parameterized object is a deleted sample dashboard -> delete+ {:card_id (:id other-card) :parameterized_object_type "dashboard"+ :parameterized_object_id (:id sample-dash) :parameter_id "dash-owner"}+ ;; non-sample source on a surviving dashboard -> keep+ {:card_id (:id other-card) :parameterized_object_type "dashboard"+ :parameterized_object_id (:id mixed-dash) :parameter_id "keep"}])]+ ;; Collection creation auto-grants an "All Users" permission row (object "/collection/<id>/") for each+ ;; sample collection; the migration must clean those up too.+ (#'custom-migrations/remove-sqlite-sample-database-on-downgrade!)+ (testing "the SQLite sample database and all of its child content are deleted, with no orphans"+ (is (not (t2/exists? :model/Database :id (:id sample))))+ (is (zero? (t2/count :metabase_table :db_id (:id sample))))+ (is (zero? (t2/count :metabase_field :table_id [:in table-ids])))+ (is (zero? (t2/count :report_card :database_id (:id sample))))+ (is (zero? (t2/count :report_dashboardcard :card_id [:in card-ids])))+ (is (zero? (t2/count :dashboardcard_series :card_id [:in card-ids]))))+ (testing "a dashboard left empty by the deletion is deleted, along with its tabs"+ (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))+ (is (zero? (t2/count :dashboard_tab :dashboard_id (:id sample-dash)))))+ (testing "parameter_card rows referencing a deleted sample card or dashboard are removed"+ (is (not (t2/exists? :model/ParameterCard :id pc-src)))+ (is (not (t2/exists? :model/ParameterCard :id pc-card-owner)))+ (is (not (t2/exists? :model/ParameterCard :id pc-dash-owner)))+ (testing "but an unrelated parameter_card is kept"+ (is (t2/exists? :model/ParameterCard :id pc-keep))))+ (testing "the sample Example collections and their permission records are deleted"+ (is (not (t2/exists? :model/Collection :id (:id examples))))+ (is (not (t2/exists? :model/Collection :id (:id ecommerce))))+ (is (zero? (t2/count :permissions :object [:in [(format "/collection/%d/" (:id examples))+ (format "/collection/%d/read/" (:id examples))]]))))+ (testing "a dashboard that still has other cards, and unrelated content, is kept"+ (is (t2/exists? :model/Dashboard :id (:id mixed-dash)))+ (is (t2/exists? :report_dashboardcard :id dc3))+ (is (t2/exists? :model/Card :id (:id other-card)))+ (is (t2/exists? :model/Database :id (:id other)))+ (is (t2/exists? :model/Collection :id (:id keep-coll))))))))++(deftest ^:mb/old-migrations-test downgrade-keeps-user-content-and-deletes-sample-dependents-test+ (testing "Downgrade deletes everything that depends on the sample DB (transitively, even user-created), then keeps+ any Example collection a user still has their own content in - leaving the RESTRICT FKs intact rather than+ aborting on them"+ (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]+ (binding [custom-migrations/*create-sample-content* true]+ (migrate!))+ (let [examples-id (:id (t2/query-one {:select [:id] :from [:collection]+ :where [:= :is_sample true] :order-by [[:id :asc]]}))+ ecommerce-id (:id (t2/query-one {:select [:id] :from [:collection]+ :where [:= :is_sample true] :order-by [[:id :desc]]}))+ sqlite-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]+ :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))+ sample-card (:id (t2/query-one {:select [:id] :from [:report_card] :where [:= :database_id sqlite-db-id] :limit 1}))+ ins (fn [t r] (first (t2/insert-returning-pks! t r)))+ other-db (ins :metabase_database {:name "User DB" :engine "h2" :is_sample false+ :details "{}" :created_at :%now :updated_at :%now})+ ;; a user question that doesn't touch the sample DB, filed in Examples -> must survive+ user-card (ins :report_card {:name "keep me" :display "table" :dataset_query "{}"+ :visualization_settings "{}" :creator_id 13371338 :database_id other-db+ :collection_id examples-id :created_at :%now :updated_at :%now})+ ;; a user model built on a sample card -> depends transitively -> must be deleted+ user-model (ins :report_card {:name "built on sample" :display "table" :dataset_query "{}"+ :visualization_settings "{}" :creator_id 13371338 :database_id other-db+ :source_card_id sample-card :collection_id examples-id+ :created_at :%now :updated_at :%now})+ ;; a curated table + a transform filed in E-commerce (RESTRICT FKs) -> keep that collection, FKs intact+ table-id (ins :metabase_table {:name "curated" :active true :db_id other-db+ :collection_id ecommerce-id :created_at :%now :updated_at :%now})+ transform-id (ins :transform {:name "tf" :source "{}" :target "{}" :source_type "query"+ :collection_id ecommerce-id :created_at :%now :updated_at :%now})+ exists? (fn [t id] (boolean (seq (t2/query {:select [1] :from [t] :where [:= :id id] :limit 1}))))+ coll-of (fn [t id] (:collection_id (t2/query-one {:select [:collection_id] :from [t] :where [:= :id id]})))]+ (migrate! :down 62)+ (testing "the sample database and everything depending on it - including the user model - are deleted"+ (is (not (exists? :metabase_database sqlite-db-id)))+ (is (not (exists? :report_card sample-card)))+ (is (not (exists? :report_card user-model))))+ (testing "a user question that does not depend on the sample DB survives"+ (is (exists? :report_card user-card)))+ (testing "Example collections holding surviving user content are kept"+ (is (exists? :collection examples-id))+ (is (exists? :collection ecommerce-id)))+ (testing "the RESTRICT-FK table and transform survive with their collection_id intact (no abort, no null-out)"+ (is (exists? :metabase_table table-id))+ (is (= ecommerce-id (coll-of :metabase_table table-id)))+ (is (exists? :transform transform-id))+ (is (= ecommerce-id (coll-of :transform transform-id))))))))++(deftest ^:mb/old-migrations-test downgrade-leaves-no-orphaned-card-children-test+ ;; B1 characterization: the migration hand-deletes only the dashcard tables and lets DB-level ON DELETE CASCADE+ ;; clear report_card's other ~11 children (query_field, query_table, card_bookmark, report_cardfavorite, ...).+ ;; Its own comment warns that MySQL 9.7 resolves cascades incompletely. This test seeds those children for a+ ;; sample card and asserts none survive the card's deletion - i.e. it FAILS if the relied-on cascade leaves+ ;; orphans. It deliberately asserts only on the outcome (no orphans), not on how the cleanup is done.+ (testing "downgrade leaves no child rows pointing at a deleted sample card"+ (impl/test-migrations ["v63.2026-06-08T00:00:00"] [migrate!]+ (binding [custom-migrations/*create-sample-content* true]+ (migrate!))+ (let [internal-user 13371338+ sqlite-db-id (:id (t2/query-one {:select [:id] :from [:metabase_database]+ :where [:and [:= :is_sample true] [:= :engine "sqlite"]]}))+ sample-card (:id (t2/query-one {:select [:id] :from [:report_card] :where [:= :database_id sqlite-db-id] :limit 1}))+ ins (fn [t r] (first (t2/insert-returning-pks! t r)))]+ ;; children of report_card that the migration leaves to ON DELETE CASCADE (not hand-deleted)+ (ins :query_field {:card_id sample-card :column "TOTAL" :explicit_reference true})+ (ins :query_table {:card_id sample-card :table "ORDERS"})+ (ins :card_bookmark {:user_id internal-user :card_id sample-card})+ (ins :report_cardfavorite {:card_id sample-card :owner_id internal-user :created_at :%now :updated_at :%now})+ (#'custom-migrations/remove-sqlite-sample-database-on-downgrade!)+ (testing "the sample card was deleted"+ (is (not (t2/exists? :report_card :id sample-card))))+ (testing "no child table is left with a row pointing at the deleted card"+ (doseq [table [:query_field :query_table :card_bookmark :report_cardfavorite]]+ (is (zero? (t2/count table :card_id sample-card))+ (str table " has an orphaned row referencing the deleted sample card"))))))))diff --git a/modules/drivers/sqlite/test/metabase/driver/sqlite_test.clj b/test/metabase/driver/sqlite_test.cljsimilarity index 100%rename from modules/drivers/sqlite/test/metabase/driver/sqlite_test.cljrename to test/metabase/driver/sqlite_test.cljdiff --git a/test/metabase/driver/util_test.clj b/test/metabase/driver/util_test.cljindex 69063cb5a2b..54f47c5d809 100644--- a/test/metabase/driver/util_test.clj+++ b/test/metabase/driver/util_test.clj@@ -486,8 +486,20 @@(testing "includes sqlite in non-hosted environment"(is (contains? (driver.u/available-drivers) :sqlite)))(mt/with-premium-features #{:hosting}- (testing "does not include sqlite in hosted environment"- (is (not (contains? (driver.u/available-drivers) :sqlite)))))))+ (testing "include sqlite in hosted environment"+ (is (contains? (driver.u/available-drivers) :sqlite))))))++(deftest sqlite-creatable-engine-test+ (testing "sqlite is always present in the engines info (the FE needs its metadata for the bundled Sample Database)"+ (testing "and is creatable off hosted Metabase"+ (mt/with-premium-features #{}+ (is (true? (get-in (driver.u/available-drivers-info) [:sqlite :creatable?])))))+ (testing "but is marked not creatable on hosted Metabase, while staying in the map"+ (mt/with-premium-features #{:hosting}+ (is (contains? (driver.u/available-drivers-info) :sqlite))+ (is (false? (get-in (driver.u/available-drivers-info) [:sqlite :creatable?])))+ (testing "other warehouse engines stay creatable"+ (is (true? (get-in (driver.u/available-drivers-info) [:postgres :creatable?]))))))))(deftest ^:parallel process-connection-prop-test(testing "process-connection-prop handles different property types"diff --git a/test/metabase/sample_data/example_content_test.clj b/test/metabase/sample_data/example_content_test.cljnew file mode 100644index 00000000000..d8bd0060301--- /dev/null+++ b/test/metabase/sample_data/example_content_test.clj@@ -0,0 +1,141 @@+(ns metabase.sample-data.example-content-test+ (:require+ [clojure.test :refer :all]+ [clojure.walk :as walk]+ [metabase.query-processor :as qp]+ [metabase.sample-data.example-content :as example-content]+ [metabase.sample-data.impl]+ [metabase.sync.core :as sync]+ [metabase.test :as mt]+ [metabase.util.json :as json]+ [toucan2.core :as t2]))++(deftest remap-blob-parameters-preserves-order-test+ (testing "remap-blob keeps a card/dashboard's parameters in their designed order rather than re-sorting by :id"+ (let [blob (json/encode [{:id "zzzz" :name "Date Range" :type "string/="}+ {:id "aaaa" :name "Vendor" :type "string/="}])+ out (json/decode+kw (#'example-content/remap-blob {} :parameters blob))]+ (is (= ["Date Range" "Vendor"] (mapv :name out))+ "parameters must stay in original order, not be alphabetized by :id"))))++(deftest remap-collection-location-test+ (let [remap #'example-content/remap-collection-location]+ (testing "root location is unchanged"+ (is (= "/" (remap {2 20} "/"))))+ (testing "each path segment is remapped independently"+ (is (= "/20/" (remap {2 20} "/2/")))+ (is (= "/20/30/" (remap {2 20, 3 30} "/2/3/"))))+ (testing "a new id colliding with a still-pending old id does not double-substitute"+ ;; old reduce-of-str/replace would turn \"/5/12/\" into \"/99/99/\"; per-segment remapping keeps them distinct+ (is (= "/12/99/" (remap {5 12, 12 99} "/5/12/"))))+ (testing "segments without a mapping keep their original id"+ (is (= "/2/7/" (remap {2 2} "/2/7/"))))))++(defn- sample-database-db []+ {:details (#'metabase.sample-data.impl/try-to-extract-sample-database! :sqlite)+ :engine :sqlite+ :name "Sample Database"})++(defn- field-ids-in-query+ "All numeric field ids referenced anywhere in a (pMBQL) dataset_query."+ [query]+ (let [ids (volatile! #{})]+ (walk/postwalk+ (fn [x]+ (when (and (vector? x) (= :field (first x)))+ (let [id (last x)]+ (when (number? id) (vswap! ids conj id))))+ x)+ query)+ @ids))++(defn- table-ids-in-query+ "All numeric :source-table ids referenced anywhere in a (pMBQL) dataset_query."+ [query]+ (let [ids (volatile! #{})]+ (walk/postwalk+ (fn [x]+ (when (and (map? x) (number? (:source-table x)))+ (vswap! ids conj (:source-table x)))+ x)+ query)+ @ids))++(deftest example-card-queries-run-after-id-bumping-upgrade-test+ (testing "After an upgrade that bumped app-db ids past the EDN-hardcoded range, every recreated example card runs.+ Cards that build on another example card carry a `\"card__N\"` :source-table; export-mbql only remaps a+ *numeric* :source-table, so these stay pinned to the EDN card id and now point at an unrelated/missing card."+ (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard+ :model/Dimension :model/Permissions]+ ;; Bump table/field ids past the EDN range (8 tables, 71 fields).+ (mt/with-temp [:model/Database other {}+ :model/Table ot {:db_id (:id other)}]+ (doseq [i (range 200)]+ (t2/insert! :model/Field {:table_id (:id ot) :name (str "filler_" i) :base_type :type/Integer+ :database_type "INT" :position i}))+ ;; Bump card ids past the EDN range (39 cards) so the EDN card ids (e.g. card__1) are taken by these+ ;; unrelated filler cards rather than by the recreated example cards.+ (dotimes [_ 60]+ (t2/insert! :model/Card (mt/with-temp-defaults :model/Card)))+ (mt/with-temp [:model/Database db (sample-database-db)]+ (sync/sync-database! db)+ (example-content/recreate-example-content! (:id db))+ (let [cards (->> (t2/select :model/Card :database_id (:id db))+ (filter (comp #{:question :model :metric} :type))+ (sort-by :id))]+ (is (seq cards) "example cards were recreated")+ (doseq [card cards]+ (testing (format "%s %s (card %s)" (:type card) (pr-str (:name card)) (:id card))+ (let [result (try (qp/process-query (:dataset_query card))+ (catch Throwable e {:error (ex-message e)}))]+ (is (= nil (:error result))+ (format "query failed: %s" (:error result))))))))))))++(deftest recreate-example-content-test+ (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard+ :model/Dimension :model/Permissions]+ ;; Pre-create an unrelated database with tables/fields so the sample DB's synced table/field ids do+ ;; NOT coincide with the EDN ids - otherwise the remap would be an identity no-op and prove nothing.+ (mt/with-temp [:model/Database other {}+ :model/Table ot {:db_id (:id other)}]+ (doseq [i (range 120)]+ (t2/insert! :model/Field {:table_id (:id ot) :name (str "filler_" i) :base_type :type/Integer+ :database_type "INT" :position i}))+ (mt/with-temp [:model/Database db (sample-database-db)]+ (sync/sync-database! db)+ (example-content/recreate-example-content! (:id db))+ (let [cards (t2/select :model/Card :database_id (:id db))+ coll-ids (set (keep :collection_id cards))+ valid-fid (set (t2/select-fn-set :id :model/Field+ {:join [[:metabase_table :t] [:= :t.id :metabase_field.table_id]]+ :where [:= :t.db_id (:id db)]}))+ valid-tid (set (t2/select-fn-set :id :model/Table :db_id (:id db)))]+ (testing "the example cards are recreated against the new sample database"+ (is (= 39 (count cards)))+ (is (every? #(= (:id db) (:database_id %)) cards)))+ (testing "every card lives in the example collection"+ (is (seq coll-ids))+ (is (every? :is_sample (t2/select :model/Collection :id [:in coll-ids]))))+ (testing "every field/table reference in every card resolves to a real column of the new database"+ (doseq [card cards]+ (let [q (:dataset_query card)]+ (doseq [fid (field-ids-in-query q)]+ (is (contains? valid-fid fid)+ (format "card %s references missing field %s" (:name card) fid)))+ (doseq [st (table-ids-in-query q)]+ (is (contains? valid-tid st)+ (format "card %s references missing table %s" (:name card) st))))))+ (testing "the dashboard and its dashcards are recreated and resolve to recreated cards"+ (let [dash (t2/select-one :model/Dashboard :collection_id [:in coll-ids])+ dashcards (t2/select :model/DashboardCard :dashboard_id (:id dash))+ card-ids (set (map :id cards))]+ (is (some? dash))+ (is (pos? (count dashcards)))+ (is (every? #(or (nil? (:card_id %)) (contains? card-ids (:card_id %))) dashcards))))+ (testing "a recreated card actually runs against the new sample database"+ (let [runnable (first (filter #(and (nil? (:source_card_id %))+ (= :question (:type %))+ (seq (table-ids-in-query (:dataset_query %))))+ cards))]+ (is (some? runnable))+ (is (pos? (count (mt/rows (qp/process-query (:dataset_query runnable)))))))))))))diff --git a/test/metabase/sample_data/impl_test.clj b/test/metabase/sample_data/impl_test.cljindex cff5964f282..1c7b0008bfe 100644--- a/test/metabase/sample_data/impl_test.clj+++ b/test/metabase/sample_data/impl_test.clj@@ -3,38 +3,39 @@(:require[clojure.core.memoize :as memoize][clojure.java.jdbc :as jdbc]- [clojure.string :as str][clojure.test :refer :all][metabase.app-db.core :as mdb]+ [metabase.config.core :as config][metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.plugins.impl :as plugins]+ [metabase.sample-data.example-content :as example-content][metabase.sample-data.impl :as sample-data][metabase.sync.core :as sync][metabase.sync.task.sync-databases-test :as task.sync-databases-test][metabase.test :as mt][metabase.util :as u]- [metabase.util.files :as u.files][metabase.warehouse-schema.models.field-values :as field-values][metabase.warehouses-rest.api-test :as api.database-test]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import (org.sqlite SQLiteException)));;; ---------------------------------------------------- Tooling -----------------------------------------------------;; These tools are pretty sophisticated for the amount of tests we have!-(defn- sample-database-db [read-write?]- ;; If `read-write?` is false, open the DB in read-only mode so we don't accidentally modify it during testing- {:details (cond-> (#'sample-data/try-to-extract-sample-database!)- (not read-write?)- (update :db #(str % ";ACCESS_MODE_DATA=r")))- :engine :h2+(defn- sample-database-db+ "Sample DB is SQLite-backed and always read-only: `try-to-extract-sample-database!` returns details with+ `:read-only? true`, which the SQLite driver honors by opening the connection in read-only `open_mode`."+ []+ {:details (#'sample-data/try-to-extract-sample-database! :sqlite)+ :engine :sqlite:name "Sample Database"})(defmacro ^:private with-temp-sample-database-db"Execute `body` with a temporary Sample Database DB bound to `db-binding`."{:style/indent 1}[[db-binding] & body]- `(mt/with-temp [:model/Database db# (sample-database-db false)]+ `(mt/with-temp [:model/Database db# (sample-database-db)](sync/sync-database! db#)(let [~db-binding db#]~@body)))@@ -51,7 +52,7 @@;;; ----------------------------------------------------- Tests -------------------------------------------------------(def ^:private extracted-db-path-regex #"^file:.*plugins/sample-database.db;USER=GUEST;PASSWORD=guest.*")+(def ^:private extracted-db-path-regex #".*plugins/sample-database\.sqlite$")(deftest extract-sample-database-test(testing "The Sample Database is copied out of the JAR into the plugins directory before the DB details are saved."@@ -59,19 +60,6 @@(with-temp-sample-database-db [db](let [db-path (get-in db [:details :db])](is (re-matches extracted-db-path-regex db-path))))))- (testing "If the plugins directory is not creatable or writable, we fall back to reading from the DB in the JAR"- (memoize/memo-clear! @#'plugins/plugins-dir*)- (let [original-var (mt/original-fn #'u.files/create-dir-if-not-exists!)]- (mt/with-dynamic-fn-redefs [u.files/create-dir-if-not-exists! (fn [_] (throw (Exception.)))]- (with-temp-sample-database-db [db]- (let [db-path (get-in db [:details :db])]- (is (not (str/includes? db-path "plugins"))))- (testing "If the plugins directory is writable on a subsequent startup, the sample DB is copied"- (with-redefs [u.files/create-dir-if-not-exists! original-var]- (memoize/memo-clear! @#'plugins/plugins-dir*)- (sample-data/update-sample-database-if-needed! db)- (let [db-path (get-in (t2/select-one :model/Database :id (:id db)) [:details :db])]- (is (re-matches extracted-db-path-regex db-path)))))))))(memoize/memo-clear! @#'plugins/plugins-dir*))(deftest sync-sample-database-test@@ -107,79 +95,154 @@(select-keys [:name :description :database_type :semantic_type :has_field_values :active :visibility_type:preview_display :display_name :fingerprint :base_type])))))))-(deftest write-rows-sample-database-test- (testing "should be able to execute INSERT, UPDATE, and DELETE statements on the Sample Database"- (mt/with-temp [:model/Database db (sample-database-db true)]+(deftest sample-database-is-read-only-test+ (testing "The Sample Database connection is read-only: reads succeed but INSERT/UPDATE/DELETE are rejected"+ (mt/with-temp [:model/Database db (sample-database-db)](sync/sync-database! db)(mt/with-db db(let [conn-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))]- (testing "update row"- (let [quantity (fn []- (->> (jdbc/query conn-spec "SELECT QUANTITY FROM ORDERS WHERE ID = 1;")- (map :quantity)))]- (testing "before"- (is (= [2]- (quantity))))- (is (= [1]- (jdbc/execute! conn-spec "UPDATE ORDERS SET QUANTITY = 1 WHERE ID = 1;")))- (testing "after"- (is (= [1]- (quantity))))- ;; TODO: this shouldn't be necessary, since we're modifying a temp sample database.- (testing "restore"- (is (= [1]- (jdbc/execute! conn-spec "UPDATE ORDERS SET QUANTITY = 2 WHERE ID = 1;"))))))- (let [rating (fn []- (->> (jdbc/query conn-spec "SELECT RATING FROM PRODUCTS WHERE PRICE = 12.345;")- (map :rating)))]- (testing "before"- (is (= []- (rating))))- (testing "insert row"- (is (= [1]- (jdbc/execute! conn-spec "INSERT INTO PRODUCTS (price, rating) VALUES (12.345, 6.789);")))- (is (= [6.789]- (rating))))- (testing "delete row"- (testing "before"- (is (= [6.789]- (rating))))- (is (= [1]- (jdbc/execute! conn-spec "DELETE FROM PRODUCTS WHERE PRICE = 12.345;")))- (testing "after"- (is (= []- (rating)))))))))))--(deftest ddl-sample-database-test- (testing "should be able to execute DDL statements on the Sample Database"- (mt/with-temp [:model/Database db (sample-database-db true)]- (sync/sync-database! db)- (mt/with-db db- (let [conn-spec (sql-jdbc.conn/db->pooled-connection-spec (mt/db))- get-tables (fn [] (set (mapv :table_name (jdbc/query conn-spec "SHOW TABLES;"))))- show-columns-from (fn [table-name] (set (mapv :field (jdbc/query conn-spec (str "SHOW COLUMNS FROM " table-name ";")))))- get-schemas (fn [] (set (mapv :schema_name (jdbc/query conn-spec "SHOW SCHEMAS;"))))]- (testing "create schema"- (is (not (contains? (get-schemas) "NEW_SCHEMA")))- (jdbc/execute! conn-spec "CREATE SCHEMA NEW_SCHEMA;")- (is (contains? (set (get-schemas)) "NEW_SCHEMA")))- (testing "drop schema"- (jdbc/execute! conn-spec "DROP SCHEMA NEW_SCHEMA;")- (is (not (contains? (get-schemas) "NEW_SCHEMA"))))- (testing "create table"- (is (not (contains? (get-tables) "NEW_TABLE")))- (jdbc/execute! conn-spec "CREATE TABLE NEW_TABLE (id INTEGER);")- (is (contains? (get-tables) "NEW_TABLE"))- (testing "add column"- (is (not (contains? (show-columns-from "NEW_TABLE") "NEW_COLUMN")))- (jdbc/execute! conn-spec "ALTER TABLE NEW_TABLE ADD COLUMN NEW_COLUMN VARCHAR(255);")- (is (contains? (show-columns-from "NEW_TABLE") "NEW_COLUMN"))- (testing "remove column"- (jdbc/execute! conn-spec "ALTER TABLE NEW_TABLE DROP COLUMN NEW_COLUMN;")- (is (not (contains? (show-columns-from "NEW_TABLE") "NEW_COLUMN")))- (testing "drop table"- (jdbc/execute! conn-spec "DROP TABLE NEW_TABLE;")- (is (not (contains? (get-tables) "NEW_TABLE"))))))))))))+ (testing "reads succeed"+ (is (= [2]+ (->> (jdbc/query conn-spec "SELECT QUANTITY FROM ORDERS WHERE ID = 1;")+ (map :quantity)))))+ (testing "writes are rejected because the connection is read-only"+ (doseq [[op sql] [["UPDATE" "UPDATE ORDERS SET QUANTITY = 1 WHERE ID = 1;"]+ ["INSERT" "INSERT INTO PRODUCTS (price, rating) VALUES (12.345, 6.789);"]+ ["DELETE" "DELETE FROM PRODUCTS WHERE PRICE = 12.345;"]]]+ (testing op+ (is (thrown-with-msg?+ SQLiteException+ #"(?i)readonly"+ (jdbc/execute! conn-spec sql)))))))))))++(deftest update-sample-database-same-engine-test+ (testing "When the bundled engine is unchanged, the sample DB is kept and only its details refreshed"+ (mt/with-temp [:model/Database db (assoc (sample-database-db)+ :is_sample true+ :details {:db "/stale/path/sample-database.sqlite"})]+ (let [extract-called? (atom false)]+ (mt/with-dynamic-fn-redefs [sample-data/extract-and-sync-sample-database! (fn [] (reset! extract-called? true))]+ (#'sample-data/update-sample-database-if-needed! db))+ (testing "the existing sample DB row is not replaced"+ (is (false? @extract-called?))+ (is (t2/exists? :model/Database :id (:id db))))+ (testing "its details were refreshed to the intended path"+ (is (re-matches extracted-db-path-regex+ (get-in (t2/select-one :model/Database :id (:id db)) [:details :db]))))))))++(deftest replace-sample-database-on-engine-change-test+ (testing "When the bundled engine changed (H2 -> SQLite), the old sample DB and its dependent content are removed"+ (mt/with-temp+ [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}+ :model/Database other-db {:engine :h2, :details {:db "mem:other"}}+ :model/Card sample-card {:database_id (:id old-sample)}+ :model/Card other-card {:database_id (:id other-db)}+ :model/Dashboard sample-dash {}+ :model/Dashboard mixed-dash {}+ :model/DashboardCard _ {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}+ :model/DashboardCard _ {:dashboard_id (:id mixed-dash), :card_id (:id sample-card)}+ :model/DashboardCard _ {:dashboard_id (:id mixed-dash), :card_id (:id other-card)}+ :model/Collection examples {:name "Examples", :is_sample true}+ :model/Collection ecommerce {:name "E-commerce", :is_sample true, :location (str "/" (:id examples) "/")}+ :model/Collection keep-coll {:name "Keep me"}]+ (let [bundled-engine (#'sample-data/sample-database-engine)+ synced-db-ids (atom [])]+ ;; recreate (collection reuse) is exercised separately; here we only check the database swap + cleanup.+ (with-redefs [example-content/recreate-example-content! (constantly nil)]+ (mt/with-dynamic-fn-redefs [sync/sync-database! (fn [db] (swap! synced-db-ids conj (:id db)) db)]+ (#'sample-data/update-sample-database-if-needed! old-sample)))+ (testing "the old sample DB is deleted, cascading to its cards"+ (is (not (t2/exists? :model/Database :id (:id old-sample))))+ (is (not (t2/exists? :model/Card :id (:id sample-card)))))+ (testing "a dashboard emptied by the deletion is deleted"+ (is (not (t2/exists? :model/Dashboard :id (:id sample-dash)))))+ (testing "a dashboard that still has other cards is kept"+ (is (t2/exists? :model/Dashboard :id (:id mixed-dash)))+ (is (t2/exists? :model/Card :id (:id other-card))))+ (testing "the Example collections are preserved (reused on recreate), not deleted by the engine swap"+ (is (t2/exists? :model/Collection :id (:id examples)))+ (is (t2/exists? :model/Collection :id (:id ecommerce)))+ (is (t2/exists? :model/Collection :id (:id keep-coll))))+ (testing "a new sample database with the bundled engine is created and synced"+ ;; replace-sample-database! inserts the new sample DB outside any with-temp, so key off the synced id+ ;; (authoritative) rather than (select-one :is_sample true), which would see leaks from other tests.+ (is (= 1 (count @synced-db-ids)) "the new database (not the old one) is what gets synced")+ (let [new-db (t2/select-one :model/Database :id (first @synced-db-ids))]+ (is (some? new-db))+ (is (not= (:id old-sample) (:id new-db)))+ (is (= bundled-engine (:engine new-db)))+ ;; clean it up so the leaked is_sample row can't pollute tests that select on is_sample.+ (t2/delete! :model/Database :id (:id new-db))))))))++(deftest replace-sample-database-cascades-to-synced-schema-test+ (testing "On H2 -> SQLite replacement, deleting the old sample DB also removes its synced schema (Tables, Fields,+ Dimensions) and dashboard tabs - i.e. every entity belonging to the old H2 sample database is gone"+ (mt/with-temp+ [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}+ :model/Table table {:db_id (:id old-sample)}+ :model/Field field {:table_id (:id table)}+ :model/Dimension dimension {:field_id (:id field)}+ :model/Card sample-card {:database_id (:id old-sample)}+ :model/Dashboard sample-dash {}+ :model/DashboardTab tab {:dashboard_id (:id sample-dash)}+ :model/DashboardCard _ {:dashboard_id (:id sample-dash), :card_id (:id sample-card)}+ :model/Collection examples {:name "Examples", :is_sample true}]+ ;; Skip recreating the replacement Sample Database (load-sample-content? false): this test only checks the+ ;; old schema cascades away, and recreating it would leak an is_sample row other tests select on.+ (with-redefs [config/load-sample-content? (constantly false)]+ (#'sample-data/update-sample-database-if-needed! old-sample))+ (testing "the old sample DB is deleted"+ (is (not (t2/exists? :model/Database :id (:id old-sample)))))+ (testing "its synced schema cascades away"+ (is (not (t2/exists? :model/Table :id (:id table))))+ (is (not (t2/exists? :model/Field :id (:id field))))+ (is (not (t2/exists? :model/Dimension :id (:id dimension)))))+ (testing "its card and the dashboard emptied by the card's removal are deleted, along with the dashboard's tabs"+ (is (not (t2/exists? :model/Card :id (:id sample-card))))+ (is (not (t2/exists? :model/Dashboard :id (:id sample-dash))))+ (is (not (t2/exists? :model/DashboardTab :id (:id tab)))))+ (testing "the Example collection is preserved (the engine swap no longer deletes it)"+ (is (t2/exists? :model/Collection :id (:id examples)))))))++(def ^:private edn-example-collection-entity-id+ "Stable entity id of the bundled Examples collection in sample-content.edn."+ "HyB3nRtqb7pBPhFG26evI")++(deftest recreate-example-content-reuses-collection-test+ (testing "recreate-example-content! reuses an existing Example collection (matched by entity id) and preserves the+ user content filed into it, instead of deleting it and creating a brand new collection"+ (mt/with-model-cleanup [:model/Collection :model/Card :model/Dashboard :model/DashboardCard+ :model/Dimension :model/Permissions]+ (with-temp-sample-database-db [db]+ (mt/with-temp+ [:model/Collection examples {:name "Examples", :is_sample true, :location "/"+ :entity_id edn-example-collection-entity-id}+ ;; a question a user filed into the Example collection - must survive the engine swap+ :model/Card user-card {:name "user question", :collection_id (:id examples), :database_id (:id db)}]+ (example-content/recreate-example-content! (:id db))+ (testing "the existing Example collection is reused, not duplicated"+ (is (t2/exists? :model/Collection :id (:id examples)))+ (is (= 1 (t2/count :model/Collection :entity_id edn-example-collection-entity-id))))+ (testing "the user's content survives, still filed in the reused Example collection"+ (is (t2/exists? :model/Card :id (:id user-card)))+ (is (= (:id examples) (:collection_id (t2/select-one :model/Card :id (:id user-card))))))+ (testing "the bundled sample content is recreated on the sample database, in the reused collection"+ (is (pos? (t2/count :model/Card :database_id (:id db))))+ (is (t2/exists? :model/Dashboard :collection_id (:id examples)))))))))++(deftest replace-sample-database-skips-content-when-sync-fails-test+ (testing "When the replacement sync fails, example content is NOT recreated - recreating against an unsynced DB+ remaps every card onto empty id maps, which is silent corruption worse than a missing example collection"+ (mt/with-model-cleanup [:model/Database]+ (mt/with-temp [:model/Database old-sample {:engine :h2, :is_sample true, :details {:db "mem:old-sample"}}]+ (let [recreate-called? (atom false)]+ (with-redefs [example-content/recreate-example-content! (fn [_] (reset! recreate-called? true))]+ (mt/with-dynamic-fn-redefs [sync/sync-database! (fn [_] (throw (ex-info "sync boom" {})))]+ (#'sample-data/update-sample-database-if-needed! old-sample)))+ (testing "recreate is skipped"+ (is (false? @recreate-called?)))+ (testing "the engine swap itself still completed - a new SQLite sample DB exists"+ (is (t2/exists? :model/Database :is_sample true :engine :sqlite))))))))(deftest sample-database-schedule-sync-test(testing "Check that the sample database has scheduled sync jobs, just like a newly created database"diff --git a/test/metabase/sample_data/sqlite_h2_parity_test.clj b/test/metabase/sample_data/sqlite_h2_parity_test.cljnew file mode 100644index 00000000000..7586987768d--- /dev/null+++ b/test/metabase/sample_data/sqlite_h2_parity_test.clj@@ -0,0 +1,110 @@+(ns metabase.sample-data.sqlite-h2-parity-test+ "Row-by-row comparison of the bundled SQLite sample database (resources/sample-database.sqlite) against+ the legacy H2 snapshot (resources/sample-database.db.mv.db) it was converted from. The two should hold+ essentially identical values.++ The data is read with the real H2 and SQLite JDBC drivers (via plain next.jdbc) but otherwise+ deliberately avoids Metabase production code (no driver multimethods, sync, or sample-data extraction):+ the point is to validate the bytes in the converted file, not the application's view of them."+ (:require+ [clojure.java.io :as io]+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing]]+ [next.jdbc :as jdbc]+ [next.jdbc.result-set :as rs]))++(set! *warn-on-reflection* true)++;; H2 returns CLOB columns as a java.sql.Clob, which is only valid while its ResultSet is open. Read the+;; characters out at column-read time (before next.jdbc closes the result set) so the value survives as a+;; plain String; otherwise reading the Clob afterwards yields garbage.+(extend-protocol rs/ReadableColumn+ java.sql.Clob+ (read-column-by-index [clob _ _] (.getSubString clob 1 (int (.length clob))))+ (read-column-by-label [clob _] (.getSubString clob 1 (int (.length clob)))))++(def ^:private tables+ ["ACCOUNTS" "ANALYTIC_EVENTS" "FEEDBACK" "INVOICES" "ORDERS" "PEOPLE" "PRODUCTS" "REVIEWS" "_METABASE_METADATA"])++(defn- order-column+ "Stable sort key so both databases yield rows in the same order. Every table has an ID except the+ key/value metadata table."+ [table]+ (if (= table "_METABASE_METADATA") "keypath" "id"))++(defn- resource-path ^String [resource]+ (.getAbsolutePath (io/as-file (io/resource resource))))++(def ^:private h2-spec+ ;; H2 wants the dbname WITHOUT the on-disk `.mv.db` suffix. Read-only; GUEST is the hardened login.+ (delay {:dbtype "h2"+ :dbname (str (str/replace (resource-path "sample-database.db.mv.db") #"\.mv\.db$" "")+ ";ACCESS_MODE_DATA=r;USER=GUEST;PASSWORD=guest")}))++(def ^:private sqlite-spec+ (delay {:dbtype "sqlite"+ :dbname (resource-path "sample-database.sqlite")}))++(defn- canonical+ "Normalize a JDBC-returned scalar so the H2 and SQLite renderings of the *same* logical value compare+ equal, while genuinely different values still differ:+ - numbers are compared numerically (H2 returns Long/Double, SQLite Integer/Double for the same column);+ - booleans are folded to 1.0/0.0 (SQLite has no boolean type, so H2 `true`/`false` lands as `1`/`0`);+ - line-break characters are dropped: the H2 source bodies use bare `\\r` separators, but the converter+ decodes CLOBs through the H2 driver's helper (which normalizes to `\\n`), so SQLite stores `\\n` while+ a direct read of the H2 CLOB still yields `\\r`. Stripping `\\r`/`\\n` compares the text content itself;+ - dates/timestamps are reduced to a canonical `yyyy-MM-dd[ HH:mm:ss[.fff]]` wall-clock string. H2 returns+ java.sql.Timestamp/Date (space separator, seconds always present); SQLite returns ISO-8601 strings+ (`T` separator, seconds and zero fractionals omitted). Seconds are padded and trailing fractional zeros+ stripped on both sides so only a real wall-clock difference survives."+ [v]+ (cond+ (nil? v) ::nil+ (boolean? v) (if v 1.0 0.0)+ (number? v) (double v)+ :else+ (let [s (str/replace (str v) #"[\r\n]" "")]+ (if-let [[_ date time frac] (re-matches #"(\d{4}-\d{2}-\d{2})(?:[ T](\d{2}:\d{2}(?::\d{2})?))?(\.\d+)?" s)]+ (str date+ (when time (str " " (cond-> time (= 5 (count time)) (str ":00"))))+ (when frac+ (let [f (str/replace frac #"0+$" "")]+ (when (not= f ".") f))))+ s))))++(defn- read-rows+ "All rows of `table` from `spec`, ordered deterministically, with every value canonicalized."+ [spec table]+ (mapv #(update-vals % canonical)+ (jdbc/execute! spec [(str "SELECT * FROM " table " ORDER BY " (order-column table))]+ {:builder-fn rs/as-unqualified-lower-maps})))++(defn- truncate [v]+ (let [s (str v)]+ (if (> (count s) 80) (str (subs s 0 80) "…") s)))++(defn- value-mismatches+ "Per-column value differences between aligned H2 and SQLite rows."+ [h2-rows sqlite-rows]+ (for [[idx h2-row sqlite-row] (map vector (range) h2-rows sqlite-rows)+ column (sort (keys h2-row))+ :let [a (get h2-row column)+ b (get sqlite-row column)]+ :when (not= a b)]+ {:row idx :id (:id h2-row) :column column+ :h2 (truncate a) :sqlite (truncate b)}))++(deftest sqlite-matches-h2-snapshot-test+ (doseq [table tables]+ (testing table+ (let [h2-rows (read-rows @h2-spec table)+ sqlite-rows (read-rows @sqlite-spec table)]+ (is (= (count h2-rows) (count sqlite-rows))+ (format "%s: row count differs (H2 %d, SQLite %d)" table (count h2-rows) (count sqlite-rows)))+ (let [mismatches (value-mismatches h2-rows sqlite-rows)]+ (is (empty? mismatches)+ (format "%s: %d value mismatch(es) across %d distinct column(s). First 10:\n%s"+ table+ (count mismatches)+ (count (distinct (map :column mismatches)))+ (str/join "\n" (map pr-str (take 10 mismatches))))))))))diff --git a/modules/drivers/sqlite/test/metabase/test/data/sqlite.clj b/test/metabase/test/data/sqlite.cljsimilarity index 100%rename from modules/drivers/sqlite/test/metabase/test/data/sqlite.cljrename to test/metabase/test/data/sqlite.cljdiff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.cljindex 3c0a6a29c45..d793d26eb08 100644--- a/test/metabase/warehouses_rest/api_test.clj+++ b/test/metabase/warehouses_rest/api_test.clj@@ -684,6 +684,40 @@{:details {:db "new"}}))(is (true? (t2/select-one-fn :is_stub :model/Database :id db-id))))))+(deftest reject-sample-database-edit-test+ (testing "PUT /api/database/:id rejects any edit to the sample database with a sample-specific message"+ (mt/with-temp [:model/Database {db-id :id} {:engine ::test-driver+ :is_sample true+ :name "Sample Database"}]+ (is (re-find #"sample database cannot be edited"+ (mt/user-http-request :crowberto :put 400 (format "database/%d" db-id)+ {:name "New Name"})))+ (testing "the row is unchanged"+ (is (= "Sample Database" (t2/select-one-fn :name :model/Database :id db-id))))+ (testing "the guard is lifted when test endpoints are enabled (e2e tests edit the sample database)"+ (mt/with-temp-env-var-value! [mb-enable-test-endpoints "true"]+ (mt/user-http-request :crowberto :put 200 (format "database/%d" db-id)+ {:name "New Name"})+ (is (= "New Name" (t2/select-one-fn :name :model/Database :id db-id))))))))++(deftest database-modifiability-flags-test+ (testing "GET /api/database/:id returns the is_sample and is_attached_dwh flags the admin UI uses to disable editing"+ (testing "sample database"+ (mt/with-temp [:model/Database {db-id :id} {:engine ::test-driver, :is_sample true}]+ (let [db (mt/user-http-request :crowberto :get 200 (format "database/%d" db-id))]+ (is (true? (:is_sample db)))+ (is (false? (:is_attached_dwh db))))))+ (testing "attached DWH"+ (mt/with-temp [:model/Database {db-id :id} {:engine ::test-driver, :is_attached_dwh true}]+ (let [db (mt/user-http-request :crowberto :get 200 (format "database/%d" db-id))]+ (is (true? (:is_attached_dwh db)))+ (is (false? (:is_sample db))))))+ (testing "ordinary database is editable (both flags false)"+ (mt/with-temp [:model/Database {db-id :id} {:engine ::test-driver}]+ (let [db (mt/user-http-request :crowberto :get 200 (format "database/%d" db-id))]+ (is (false? (:is_sample db)))+ (is (false? (:is_attached_dwh db))))))))+(deftest update-database-provider-name-test(testing "PUT /api/database/:id"(testing "should be able to set and unset `provider_name`"