Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Transform tests batch 2 (#82770)

Alexander Polyankin74216b30981d
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 1c47ea4e99a..9606cf6dd37 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2328,7 +2328,8 @@
{:team "UX West"
:api #{metabase.revisions.api
metabase.revisions.core
- metabase.revisions.init}
+ metabase.revisions.init
+ metabase.revisions.models.revision}
:uses #{api
collections
dashboards
@@ -2358,7 +2359,8 @@
:model/Measure
:model/ParameterCard
:model/Segment
- :model/Transform}}
+ :model/Transform
+ :model/TransformTest}}
run-tracking
{:team "Gadget"
@@ -4357,14 +4359,18 @@
{:team "Graphy"
:api #{metabase-enterprise.transform-testing.api metabase-enterprise.transform-testing.init}
:uses #{api
+ audit-app
driver
events
lib
lib.schema
models
premium-features
+ revisions
+ run-tracking
sql-tools
sql-tools.parsing
+ task
transforms.base
util
warehouses}
diff --git a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
index 2612bb681a7..44cafccea21 100644
--- a/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/serialization/v2/models.clj
@@ -170,6 +170,7 @@
"TransformJobRun"
"TransformRun"
"TransformRunCancelation"
+ "TransformTestRun"
"Undo"
"User"
"UserKeyValue"
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj
index 26c878aeb00..f4ab758421b 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/db.clj
@@ -28,6 +28,24 @@
[:creator_id ::lib.schema.id/user]]]]
(t2/insert-returning-instance! :model/TransformTest transform-test))
+(mu/defn insert-transform-test-run! :- ::transform-testing.schema/transform-test-run
+ "Insert a started TransformTestRun for `transform-test-id` and return it."
+ [transform-test-id :- ms/PositiveInt
+ initiated-by :- [:maybe ::lib.schema.id/user]]
+ (t2/insert-returning-instance! :model/TransformTestRun
+ {:transform_test_id transform-test-id
+ :initiated_by initiated-by
+ :status :started}))
+
+(mu/defn finish-started-transform-test-run!
+ "Set `status` and the end time of TransformTestRun `run-id` if it is still started."
+ [run-id :- ms/PositiveInt
+ status :- ::transform-testing.schema/run-status]
+ (t2/update! :model/TransformTestRun
+ :id run-id
+ :status "started"
+ {:status status :end_time :%now}))
+
(mu/defn update-transform-test!
"Apply `updates` to the TransformTest with `id`."
[id :- ms/PositiveInt
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/init.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/init.clj
index aa62d78e8ed..49f767677ab 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/init.clj
@@ -1,4 +1,6 @@
(ns metabase-enterprise.transform-testing.init
(:require
[metabase-enterprise.transform-testing.api]
- [metabase-enterprise.transform-testing.models]))
+ [metabase-enterprise.transform-testing.models]
+ [metabase-enterprise.transform-testing.models.transform-test-run]
+ [metabase-enterprise.transform-testing.run-tracking]))
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj
index 566a588e99a..66c232d9487 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/models.clj
@@ -2,11 +2,13 @@
(:require
[metabase-enterprise.transform-testing.db :as transform-testing.db]
[metabase-enterprise.transform-testing.schema :as transform-testing.schema]
+ [metabase.audit-app.core :as audit-app]
[metabase.events.core :as events]
[metabase.lib.core :as lib]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
[metabase.premium-features.core :refer [defenterprise]]
+ [metabase.revisions.models.revision :as revision]
[metabase.util.malli :as mu]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -33,6 +35,10 @@
{:inputs (json-column ::transform-testing.schema/inputs)
:expectations (json-column ::transform-testing.schema/expectations)})
+(defmethod audit-app/model-details :model/TransformTest
+ [transform-test _event-type]
+ (select-keys transform-test [:transform_id :name :description]))
+
;;; ------------------------------------------------- Permissions --------------------------------------------------
(defmethod mi/can-read? :model/TransformTest
@@ -53,16 +59,36 @@
[_model instance]
(mi/can-write? :model/Transform (:transform_id instance)))
+;;; -------------------------------------------------- Revisions ---------------------------------------------------
+
+(def ^:private revision-columns
+ [:transform_id :name :description :inputs :expectations])
+
+(defmethod revision/serialize-instance :model/TransformTest
+ [_model _id transform-test]
+ (select-keys transform-test revision-columns))
+
+(defmethod revision/revision-readable? :model/TransformTest
+ [_model object]
+ (mi/can-read? (t2/instance :model/TransformTest object)))
+
+(defmethod revision/revert-to-revision! :model/TransformTest
+ [_model id _user-id serialized-instance]
+ (transform-testing.db/update-transform-test! id (select-keys serialized-instance revision-columns)))
+
;;; ---------------------------------------------------- Events ----------------------------------------------------
(t2/define-after-insert :model/TransformTest [transform-test]
(when-not mi/*deserializing?*
- (events/publish-event! :event/transform-test-create {:object transform-test}))
+ (events/publish-event! :event/transform-test-create
+ {:object (t2/instance :model/TransformTest transform-test)
+ :user-id (:creator_id transform-test)}))
transform-test)
(t2/define-after-update :model/TransformTest [transform-test]
(when-not mi/*deserializing?*
- (events/publish-event! :event/transform-test-update {:object transform-test}))
+ (events/publish-event! :event/transform-test-update
+ {:object (t2/instance :model/TransformTest transform-test)}))
transform-test)
(defenterprise delete-transform-tests!
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/models/transform_test_run.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/models/transform_test_run.clj
new file mode 100644
index 00000000000..f80059e6dea
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/models/transform_test_run.clj
@@ -0,0 +1,17 @@
+(ns metabase-enterprise.transform-testing.models.transform-test-run
+ (:require
+ [metabase.audit-app.core :as audit-app]
+ [metabase.models.interface :as mi]
+ [methodical.core :as methodical]
+ [toucan2.core :as t2]))
+
+(methodical/defmethod t2/table-name :model/TransformTestRun [_model] :transform_test_run)
+
+(derive :model/TransformTestRun :metabase/model)
+
+(defmethod audit-app/model-details :model/TransformTestRun
+ [transform-test-run _event-type]
+ (select-keys transform-test-run [:transform_test_id :status]))
+
+(t2/deftransforms :model/TransformTestRun
+ {:status mi/transform-keyword})
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/run_tracking.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/run_tracking.clj
new file mode 100644
index 00000000000..32126c98de4
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/run_tracking.clj
@@ -0,0 +1,84 @@
+(ns metabase-enterprise.transform-testing.run-tracking
+ "Cluster-safe lifecycle tracking for transform test runs."
+ (:require
+ [medley.core :as m]
+ [metabase-enterprise.transform-testing.db :as transform-testing.db]
+ [metabase-enterprise.transform-testing.schema :as transform-testing.schema]
+ [metabase.events.core :as events]
+ [metabase.lib.schema.id :as lib.schema.id]
+ [metabase.run-tracking.core :as rt]
+ [metabase.task.core :as task]
+ [metabase.util.log :as log]
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.schema :as ms]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private heartbeat-stale-minutes 5)
+
+(defonce ^:private active-runs
+ (atom #{}))
+
+(mu/defn start-run! :- ::transform-testing.schema/transform-test-run
+ "Create a started run for `transform-test-id` and register it as owned by this process."
+ ([transform-test-id :- ms/PositiveInt]
+ (start-run! transform-test-id nil))
+ ([transform-test-id :- ms/PositiveInt
+ initiated-by :- [:maybe ::lib.schema.id/user]]
+ (let [run (transform-testing.db/insert-transform-test-run! transform-test-id initiated-by)]
+ (events/publish-event! :event/transform-test-run-start
+ (-> {:object run}
+ (m/assoc-some :user-id initiated-by)))
+ (swap! active-runs conj (:id run))
+ run)))
+
+(mu/defn finish-run!
+ "Finish `run-id` if it is still started, then release this process's ownership.
+
+ The status guard prevents a late completion from overwriting a timeout assigned by the cluster reaper."
+ [run-id :- ms/PositiveInt
+ status :- ::transform-testing.schema/run-status]
+ (try
+ (transform-testing.db/finish-started-transform-test-run! run-id status)
+ (finally
+ (swap! active-runs disj run-id))))
+
+(defn heartbeat-runs!
+ "Stamp a heartbeat on the given runs while they are still started."
+ [run-ids]
+ (rt/heartbeat-ids! :model/TransformTestRun [:status "started"] :last_heartbeat run-ids))
+
+(defn heartbeat-and-reconcile-runs!
+ "Heartbeat the runs owned by this process and forget any that became terminal elsewhere."
+ []
+ (rt/heartbeat-and-reconcile! {:model :model/TransformTestRun
+ :active [:status "started"]
+ :ids @active-runs
+ :heartbeat! heartbeat-runs!
+ :on-gone #(swap! active-runs disj %)}))
+
+(defn reap-orphaned-runs!
+ "Time out started runs whose owning process has stopped heartbeating."
+ [stale-minutes]
+ (let [reaped (rt/reap-orphaned! {:model :model/TransformTestRun
+ :active [:status "started"]
+ :stale [{:column :last_heartbeat :age stale-minutes :unit :minute}]
+ :terminal {:status "timeout" :end_time :%now}})]
+ (doseq [run reaped]
+ (try
+ (events/publish-event! :event/transform-test-run-timeout
+ (-> {:object (assoc run :status :timeout)}
+ (m/assoc-some :user-id (:initiated_by run))))
+ (catch Throwable t
+ (log/warnf "Failed to publish transform-test-run-timeout event for run %s: %s"
+ (:id run)
+ (ex-message t)))))
+ reaped))
+
+(defmethod task/init! ::TransformTestRunHeartbeat [_]
+ (rt/start-heartbeat! heartbeat-and-reconcile-runs! 1))
+
+(defmethod task/init! ::TransformTestRunReaper [_]
+ (rt/schedule-reaper! {:job-key "metabase.transform-testing.reaper"
+ :label "transform test run"
+ :reap-fn #(reap-orphaned-runs! heartbeat-stale-minutes)}))
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj
index d2d26320edc..28c3ee84b27 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/runner.clj
@@ -27,11 +27,11 @@
[metabase-enterprise.transform-testing.expectations.empty]
[metabase-enterprise.transform-testing.expectations.equals]
[metabase-enterprise.transform-testing.expectations.protocol :as expectations.protocol]
+ [metabase-enterprise.transform-testing.run-tracking :as transform-testing.run-tracking]
[metabase-enterprise.transform-testing.schema :as transform-testing.schema]
[metabase-enterprise.transform-testing.validator :as transform-testing.validator]
[metabase.api.common :as api]
[metabase.driver :as driver]
- [metabase.driver.sql.normalize :as sql.normalize]
[metabase.driver.util :as driver.u]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.sql-parsing.core :as sql-parsing]
@@ -205,7 +205,7 @@
source (parsed-source driver transform transform-id)
input->table (into {} (map (fn [input] [input (driver/temp-table-name driver)])) inputs)
output-table (driver/temp-table-name driver)
- default-schema (sql.normalize/default-schema driver database)
+ default-schema (:default_schema database)
replacements (transform-testing.compile/table-replacements transform input->table output-table default-schema)
compiled (transform-testing.compile/compile-transform driver source replacements)]
(transform-testing.validator/validate
@@ -241,21 +241,31 @@
Throws a typed refusal from [[metabase-enterprise.transform-testing.errors]] when the run cannot happen. A
failing expectation is not a refusal: it rides back as that expectation's own result."
- [{:keys [transform_id inputs expectations]} :- ::transform-testing.schema/transform-test]
+ [{:keys [id transform_id inputs expectations]} :- ::transform-testing.schema/transform-test]
(let [{:keys [driver database input->table output-table labels replacements compiled]}
- (validated-plan transform_id inputs expectations)]
- (driver/do-with-test-connection
- driver database
- (fn [conn]
- (try
- (create-inputs! driver conn input->table labels)
- (create-output! driver conn output-table compiled labels)
- (let [results (check-expectations driver conn
- {:driver driver :output-table output-table :replacements replacements}
- expectations labels)]
- {:status (if (every? #(= :passed (:status %)) results) :passed :failed)
- :expectations results
- :tables labels})
- (finally
- (doseq [table (cons output-table (vals input->table))]
- (transform-testing.executor/drop-temp-table! driver conn table))))))))
+ (validated-plan transform_id inputs expectations)
+ {run-id :id} (transform-testing.run-tracking/start-run! id api/*current-user-id*)
+ final-status (atom :error)]
+ (try
+ (let [result
+ (driver/do-with-test-connection
+ driver database
+ (fn [conn]
+ (try
+ (create-inputs! driver conn input->table labels)
+ (create-output! driver conn output-table compiled labels)
+ (let [results (check-expectations driver conn
+ {:driver driver
+ :output-table output-table
+ :replacements replacements}
+ expectations labels)]
+ {:status (if (every? #(= :passed (:status %)) results) :passed :failed)
+ :expectations results
+ :tables labels})
+ (finally
+ (doseq [table (cons output-table (vals input->table))]
+ (transform-testing.executor/drop-temp-table! driver conn table))))))]
+ (reset! final-status (:status result))
+ result)
+ (finally
+ (transform-testing.run-tracking/finish-run! run-id @final-status)))))
diff --git a/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj b/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj
index 93d3f19eff0..c02654663c0 100644
--- a/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj
+++ b/enterprise/backend/src/metabase_enterprise/transform_testing/schema.clj
@@ -169,6 +169,22 @@
[:inputs {:optional true} ::inputs]
[:expectations {:optional true} ::expectations]])
+(mr/def ::run-status
+ "The persisted lifecycle state of a transform test run."
+ [:enum {:decode/normalize lib.schema.common/normalize-keyword}
+ :started :passed :failed :error :timeout])
+
+(mr/def ::transform-test-run
+ "A persisted execution of a transform test."
+ [:map {:closed true}
+ [:id ms/PositiveInt]
+ [:transform_test_id [:maybe ms/PositiveInt]]
+ [:initiated_by [:maybe ::lib.schema.id/user]]
+ [:start_time ms/TemporalInstant]
+ [:end_time [:maybe ms/TemporalInstant]]
+ [:status ::run-status]
+ [:last_heartbeat ms/TemporalInstant]])
+
(mr/def ::status
"The outcome of a transform test run, or of one of its expectations.
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
index 715a24cc956..7b2745f85a3 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/native_validation_test.clj
@@ -486,7 +486,7 @@
(deftest ^:parallel placeholder-no-card-refs-test
(testing "Queries with tables named as placeholder produce expected validation results"
(let [mp (lib.tu/mock-metadata-provider
- {:database {:id 100 :engine :h2}
+ {:database {:id 100 :engine :h2 :default-schema "PUBLIC"}
:tables [{:name "MB__VALIDAT_CARD__10330"
:schema "PUBLIC"
:id 333
diff --git a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
index bd943a50660..76f1959b6d3 100644
--- a/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/serialization/models/entity_id_test.clj
@@ -137,6 +137,8 @@
:model/TaskRun
:model/Tenant
:model/TimelineEvent
+ ;; Run history is intentionally not serialized.
+ :model/TransformTestRun
;; TODO we should remove these models from here once serialization is supported
:model/TransformRun
:model/TransformRunCancelation
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/audit_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/audit_test.clj
new file mode 100644
index 00000000000..85766cabd5c
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/audit_test.clj
@@ -0,0 +1,87 @@
+(ns metabase-enterprise.transform-testing.audit-test
+ (:require
+ [clojure.test :refer [deftest is testing use-fixtures]]
+ [metabase-enterprise.transform-testing.models]
+ [metabase-enterprise.transform-testing.models.transform-test-run]
+ [metabase-enterprise.transform-testing.run-tracking :as transform-testing.run-tracking]
+ [metabase.audit-app.events.audit-log]
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
+ (:import
+ (java.time OffsetDateTime ZoneOffset)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once
+ (fn [test-fn]
+ (mt/with-premium-features #{:audit-app}
+ (test-fn))))
+
+(defn- minutes-ago ^OffsetDateTime [^long n]
+ (.minusMinutes (OffsetDateTime/now ZoneOffset/UTC) n))
+
+(deftest transform-test-audit-events-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id
+ :creator_id (mt/user->id :crowberto)
+ :name "Original"
+ :description "Checks output"}]
+ (testing "create"
+ (is (= {:model_id transform-test-id
+ :user_id (mt/user->id :crowberto)
+ :details {:transform_id transform-id
+ :name "Original"
+ :description "Checks output"}
+ :topic :transform-test-create
+ :model "TransformTest"}
+ (mt/latest-audit-log-entry :transform-test-create transform-test-id))))
+ (testing "update"
+ (t2/update! :model/TransformTest transform-test-id {:name "Renamed"})
+ (is (= {:model_id transform-test-id
+ :user_id (mt/user->id :crowberto)
+ :details {:transform_id transform-id
+ :name "Renamed"
+ :description "Checks output"}
+ :topic :transform-test-update
+ :model "TransformTest"}
+ (mt/latest-audit-log-entry :transform-test-update transform-test-id))))
+ (testing "delete"
+ (t2/delete! :model/TransformTest :id transform-test-id)
+ (is (= {:model_id transform-test-id
+ :user_id (mt/user->id :crowberto)
+ :details {:transform_id transform-id
+ :name "Renamed"
+ :description "Checks output"}
+ :topic :transform-test-delete
+ :model "TransformTest"}
+ (mt/latest-audit-log-entry :transform-test-delete transform-test-id)))))))
+
+(deftest transform-test-run-audit-events-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id}]
+ (let [initiated-by (mt/user->id :rasta)
+ {run-id :id} (transform-testing.run-tracking/start-run! transform-test-id initiated-by)]
+ (try
+ (testing "start"
+ (is (= {:model_id run-id
+ :user_id initiated-by
+ :details {:transform_test_id transform-test-id
+ :status "started"}
+ :topic :transform-test-run-start
+ :model "TransformTestRun"}
+ (mt/latest-audit-log-entry :transform-test-run-start run-id))))
+ (testing "cluster reaping records the timeout"
+ (t2/update! :model/TransformTestRun run-id {:last_heartbeat (minutes-ago 10)})
+ (is (= [run-id] (mapv :id (transform-testing.run-tracking/reap-orphaned-runs! 5))))
+ (is (= {:model_id run-id
+ :user_id initiated-by
+ :details {:transform_test_id transform-test-id
+ :status "timeout"}
+ :topic :transform-test-run-timeout
+ :model "TransformTestRun"}
+ (mt/latest-audit-log-entry :transform-test-run-timeout run-id))))
+ (finally
+ (transform-testing.run-tracking/finish-run! run-id :timeout)
+ (t2/delete! :model/TransformTestRun :id run-id)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/models_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/models_test.clj
index 8d761af3a04..4dce2317c01 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/models_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/models_test.clj
@@ -1,13 +1,15 @@
(ns metabase-enterprise.transform-testing.models-test
- "The `:model/TransformTest` column transforms and delete hook. Both JSON columns normalize and
- validate on the way in and normalize on the way out, so every reader gets values the schema has
- passed — including a serdes import, which goes through Toucan and never touches an API endpoint."
+ "Transform testing model behavior: TransformTest column transforms and deletion, plus the
+ TransformTestRun heartbeat and terminal-state lifecycle."
(:require
[clojure.test :refer [deftest is testing]]
[metabase-enterprise.transform-testing.expectations.protocol :as expectations.protocol]
+ [metabase-enterprise.transform-testing.run-tracking :as transform-testing.run-tracking]
[metabase.events.core :as events]
[metabase.test :as mt]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.time OffsetDateTime ZoneOffset)))
(set! *warn-on-reflection* true)
@@ -210,3 +212,71 @@
(comp (filter (comp #{:event/transform-test-delete} first))
(map (comp :id :object second)))
@published))))))))
+
+;;; --------------------------------------------- Run tracking ---------------------------------------------
+
+(defn- minutes-ago ^OffsetDateTime [^long n]
+ (.minusMinutes (OffsetDateTime/now ZoneOffset/UTC) n))
+
+(defn- recent?
+ [timestamp]
+ (.isAfter ^OffsetDateTime timestamp (minutes-ago 1)))
+
+(deftest transform-test-run-lifecycle-test
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id}]
+ (let [initiated-by (mt/user->id :crowberto)
+ {run-id :id :as started} (transform-testing.run-tracking/start-run! transform-test-id initiated-by)]
+ (testing "a new run is started with database timestamps"
+ (is (= :started (:status started)))
+ (is (= initiated-by (:initiated_by started)))
+ (is (some? (:start_time started)))
+ (is (nil? (:end_time started)))
+ (is (some? (:last_heartbeat started))))
+ (testing "the owning process heartbeats its registered run"
+ (t2/update! :model/TransformTestRun run-id {:last_heartbeat (minutes-ago 10)})
+ (transform-testing.run-tracking/heartbeat-and-reconcile-runs!)
+ (is (recent? (t2/select-one-fn :last_heartbeat :model/TransformTestRun :id run-id))))
+ (testing "finishing sets a terminal status and end time"
+ (is (= 1 (transform-testing.run-tracking/finish-run! run-id :passed)))
+ (let [finished (t2/select-one :model/TransformTestRun :id run-id)]
+ (is (= :passed (:status finished)))
+ (is (some? (:end_time finished))))))))
+
+(deftest reap-orphaned-transform-test-runs-test
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id}
+ :model/TransformTestRun {stale-id :id} {:transform_test_id transform-test-id
+ :last_heartbeat (minutes-ago 10)}
+ :model/TransformTestRun {fresh-id :id} {:transform_test_id transform-test-id
+ :last_heartbeat (minutes-ago 1)}]
+ (testing "only a stale started run is reaped"
+ (is (= [stale-id]
+ (mapv :id (transform-testing.run-tracking/reap-orphaned-runs! 5))))
+ (is (= :timeout (t2/select-one-fn :status :model/TransformTestRun :id stale-id)))
+ (is (some? (t2/select-one-fn :end_time :model/TransformTestRun :id stale-id)))
+ (is (= :started (t2/select-one-fn :status :model/TransformTestRun :id fresh-id))))
+ (testing "a late completion cannot overwrite the cluster reaper's timeout"
+ (is (zero? (transform-testing.run-tracking/finish-run! stale-id :passed)))
+ (is (= :timeout (t2/select-one-fn :status :model/TransformTestRun :id stale-id))))
+ (testing "a second sweep is idempotent"
+ (is (empty? (transform-testing.run-tracking/reap-orphaned-runs! 5))))))
+
+(deftest deleting-transform-test-preserves-runs-test
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id}]
+ (let [{run-id :id} (transform-testing.run-tracking/start-run! transform-test-id)]
+ (t2/delete! :model/TransformTest transform-test-id)
+ (is (nil? (t2/select-one-fn :transform_test_id :model/TransformTestRun :id run-id)))
+ (is (= 1 (transform-testing.run-tracking/finish-run! run-id :passed)))
+ (is (= :passed (t2/select-one-fn :status :model/TransformTestRun :id run-id))))))
+
+(deftest deleting-run-initiator-preserves-run-test
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id}
+ :model/User {user-id :id} {}]
+ (let [{run-id :id} (transform-testing.run-tracking/start-run! transform-test-id user-id)]
+ (t2/delete! :model/User :id user-id)
+ (is (nil? (t2/select-one-fn :initiated_by :model/TransformTestRun :id run-id)))
+ (is (= 1 (transform-testing.run-tracking/finish-run! run-id :passed)))
+ (is (= :passed (t2/select-one-fn :status :model/TransformTestRun :id run-id))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj
index df16818c056..8024737ade3 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/remapping_test.clj
@@ -18,8 +18,8 @@
;; Guard B parses the rewritten SQL through the namespace that registers its result schema.
(comment metabase.sql-tools.macaw.references/keep-me)
-;; `sql.normalize/default-schema`, which decides whether a bare table key is registered, dispatches
-;; on an initialized driver. These tests parse and rewrite SQL only — no connection is needed.
+;; The Database's synced `default_schema` decides whether a bare table key is registered. Identifier normalization
+;; still dispatches on an initialized driver; these tests parse and rewrite SQL only, so no connection is needed.
(use-fixtures :once (fn [f] (driver/the-initialized-driver :h2) (f)))
;;; -------------------------------------------- Harness --------------------------------------------
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/revision_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/revision_test.clj
new file mode 100644
index 00000000000..8fe8ab350ef
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/revision_test.clj
@@ -0,0 +1,84 @@
+(ns metabase-enterprise.transform-testing.revision-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [metabase-enterprise.transform-testing.models]
+ [metabase-enterprise.transform-testing.models.transform-test-run]
+ [metabase-enterprise.transform-testing.run-tracking :as transform-testing.run-tracking]
+ [metabase.revisions.core :as revisions]
+ [metabase.revisions.events]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(deftest transform-test-revision-serialization-test
+ (let [transform-test (t2/instance :model/TransformTest
+ {:id 1
+ :entity_id "test-entity-id"
+ :transform_id 2
+ :creator_id 3
+ :name "No nulls"
+ :description "Checks the output"
+ :inputs []
+ :expectations []
+ :created_at :not-revisioned
+ :updated_at :not-revisioned})]
+ (is (= {:transform_id 2
+ :name "No nulls"
+ :description "Checks the output"
+ :inputs []
+ :expectations []}
+ (revisions/serialize-instance :model/TransformTest 1 transform-test)))))
+
+(deftest transform-test-create-and-update-revisions-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id
+ :creator_id (mt/user->id :crowberto)
+ :name "Original"}]
+ (testing "creation records the initial revision"
+ (let [[creation] (revisions/revisions :model/TransformTest transform-test-id)]
+ (is (true? (:is_creation creation)))
+ (is (= "Original" (get-in creation [:object :name])))
+ (is (= (mt/user->id :crowberto) (:user_id creation)))))
+ (testing "an update records the changed editable fields"
+ (t2/update! :model/TransformTest transform-test-id {:name "Renamed"})
+ (let [[updated creation] (revisions/revisions :model/TransformTest transform-test-id)]
+ (is (= "Renamed" (get-in updated [:object :name])))
+ (is (false? (:is_creation updated)))
+ (is (= "Original" (get-in creation [:object :name]))))))))
+
+(deftest transform-test-revision-revert-test
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-temporary-raw-setting-values [transforms-enabled "true"]
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id
+ :name "Original"}]
+ (t2/update! :model/TransformTest transform-test-id {:name "Renamed"})
+ (let [creation (last (revisions/revisions :model/TransformTest transform-test-id))]
+ (mt/user-http-request :crowberto :post 200 "revision/revert"
+ {:entity "transform-test"
+ :id transform-test-id
+ :revision_id (:id creation)})
+ (testing "the saved test is restored"
+ (is (= "Original" (t2/select-one-fn :name :model/TransformTest :id transform-test-id))))
+ (testing "the reversion is recorded after the model update revision"
+ (let [[reversion reverted-update update creation]
+ (revisions/revisions :model/TransformTest transform-test-id)]
+ (is (= 4 (count (revisions/revisions :model/TransformTest transform-test-id))))
+ (is (true? (:is_reversion reversion)))
+ (is (= "Original" (get-in reversion [:object :name])))
+ (is (false? (:is_reversion reverted-update)))
+ (is (= "Original" (get-in reverted-update [:object :name])))
+ (is (= "Renamed" (get-in update [:object :name])))
+ (is (true? (:is_creation creation)))))))))))
+
+(deftest transform-test-runs-are-not-revisioned-test
+ (mt/with-current-user (mt/user->id :crowberto)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformTest {transform-test-id :id} {:transform_id transform-id}]
+ (let [{run-id :id} (transform-testing.run-tracking/start-run! transform-test-id (mt/user->id :crowberto))]
+ (try
+ (transform-testing.run-tracking/finish-run! run-id :passed)
+ (is (zero? (t2/count :model/Revision :model "TransformTestRun" :model_id run-id)))
+ (finally
+ (t2/delete! :model/TransformTestRun :id run-id)))))))
diff --git a/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj b/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj
index 590ad93cb91..09990c9e289 100644
--- a/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transform_testing/runner_test.clj
@@ -4,10 +4,12 @@
[metabase-enterprise.transform-testing.errors :as transform-testing.errors]
[metabase-enterprise.transform-testing.runner :as transform-testing.runner]
[metabase-enterprise.transform-testing.test-util :as transform-testing.test-util]
+ [metabase.api.common :as api]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.sql-tools.settings :as sql-tools.settings]
- [metabase.test :as mt])
+ [metabase.test :as mt]
+ [toucan2.core :as t2])
(:import
(clojure.lang ExceptionInfo)))
@@ -84,27 +86,72 @@
[{:name "id" :cast_type int-type}
{:name "name" :cast_type text-type}]))
+(defn- check-empty-expectation!
+ [schema table transform-id description where expected]
+ (let [result (run-test! schema table transform-id one-row
+ [{:type :empty
+ :name description
+ :sql (str "SELECT * FROM " schema ".people_summary WHERE " where)}])]
+ (is (= expected (:status result)))
+ (is (= [{:name description :type :empty :status expected}]
+ (mapv #(select-keys % [:name :type :status]) (:expectations result))))))
+
(deftest run-transform-test-empty-expectation-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
(with-people-transform
(fn [schema table transform-id]
- (doseq [[desc where expected] [["passes when the query returns no rows"
- "name IS NULL"
- :passed]
- ["reads the input test data instead of the input table"
- "id <> 1"
- :passed]
- ["fails when the query returns rows"
- "name = 'abc'"
- :failed]]]
- (testing desc
- (let [result (run-test! schema table transform-id one-row
- [{:type :empty
- :name desc
- :sql (str "SELECT * FROM " schema ".people_summary WHERE " where)}])]
- (is (= expected (:status result)))
- (is (= [{:name desc :type :empty :status expected}]
- (mapv #(select-keys % [:name :type :status]) (:expectations result)))))))))))
+ (testing "passes when the query returns no rows"
+ (check-empty-expectation! schema table transform-id "no rows" "name IS NULL" :passed))
+ (testing "reads the input test data instead of the input table"
+ (check-empty-expectation! schema table transform-id "input test data" "id <> 1" :passed))
+ (testing "fails when the query returns rows"
+ (check-empty-expectation! schema table transform-id "rows returned" "name = 'abc'" :failed))))))
+
+(defn- check-persisted-run!
+ [schema table transform-id where expected]
+ (mt/with-temp [:model/TransformTest {transform-test-id :id :as transform-test}
+ {:transform_id transform-id
+ :inputs [{:table {:schema schema :name table}
+ :format :sql
+ :sql one-row}]
+ :expectations [{:type :empty
+ :name "check"
+ :sql (str "SELECT * FROM " schema ".people_summary WHERE " where)}]}]
+ (binding [api/*current-user-id* (mt/user->id :crowberto)]
+ (is (= expected (:status (transform-testing.runner/run-transform-test! transform-test)))))
+ (let [run (t2/select-one :model/TransformTestRun :transform_test_id transform-test-id)]
+ (is (= expected (:status run)))
+ (is (= (mt/user->id :crowberto) (:initiated_by run)))
+ (is (some? (:start_time run)))
+ (is (some? (:end_time run))))))
+
+(deftest run-transform-test-persists-passed-run-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
+ (with-people-transform
+ (fn [schema table transform-id]
+ (check-persisted-run! schema table transform-id "name IS NULL" :passed)))))
+
+(deftest run-transform-test-persists-failed-run-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
+ (with-people-transform
+ (fn [schema table transform-id]
+ (check-persisted-run! schema table transform-id "id = 1" :failed)))))
+
+(deftest run-transform-test-persists-post-validation-error-test
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
+ (with-people-transform
+ (fn [schema table transform-id]
+ (mt/with-temp [:model/TransformTest {transform-test-id :id :as transform-test}
+ {:transform_id transform-id
+ :inputs [{:table {:schema schema :name table}
+ :format :sql
+ :sql "SELECT * FROM transform_test_input_that_does_not_exist"}]
+ :expectations []}]
+ (is (thrown? ExceptionInfo
+ (transform-testing.runner/run-transform-test! transform-test)))
+ (let [run (t2/select-one :model/TransformTestRun :transform_test_id transform-test-id)]
+ (is (= :error (:status run)))
+ (is (some? (:end_time run)))))))))
(deftest run-transform-test-equals-passes-test
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/testing)
@@ -198,7 +245,10 @@
(is (some? ex)))
(testing "with a typed refusal that names the undeclared table"
(is (= ::transform-testing.errors/missing-inputs (:error-type (ex-data ex))))
- (is (re-find (re-pattern (str "(?i)" table)) (ex-message ex))))))))))
+ (is (re-find (re-pattern (str "(?i)" table)) (ex-message ex))))
+ (testing "without creating a run"
+ (is (false? (t2/exists? :model/TransformTestRun
+ :transform_test_id (:id transform-test)))))))))))
(deftest run-transform-test-rejects-partially-declared-inputs-test
(testing "Guard A: a join declaring only some of its input tables is rejected, naming the gap,"
diff --git a/frontend/src/metabase-types/api/database.ts b/frontend/src/metabase-types/api/database.ts
index b535fb44c40..37cd55bbde8 100644
--- a/frontend/src/metabase-types/api/database.ts
+++ b/frontend/src/metabase-types/api/database.ts
@@ -65,6 +65,7 @@ export interface Database extends DatabaseData {
features?: DatabaseFeature[];
creator_id?: number;
timezone?: string;
+ default_schema?: string | null;
native_permissions: "write" | "none";
transforms_permissions?: "write" | "none";
initial_sync_status: InitialSyncStatus;
diff --git a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
index fa60f3c22b7..5da320fa1a4 100644
--- a/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
+++ b/modules/drivers/bigquery-cloud-sdk/test/metabase/driver/bigquery_cloud_sdk_test.clj
@@ -13,6 +13,7 @@
[metabase.driver.bigquery-cloud-sdk.common :as bigquery.common]
[metabase.driver.common.table-rows-sample :as table-rows-sample]
[metabase.driver.settings :as driver.settings]
+ [metabase.driver.sql :as driver.sql]
[metabase.driver.sync :as driver.s]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
@@ -40,6 +41,10 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :bigquery-cloud-sdk
+ (is (nil? (driver.sql/default-schema :bigquery-cloud-sdk (mt/db))))))
+
(defn ^:private get-test-data-name
[]
(bigquery.tx/test-dataset-id
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index 29af85f395d..cc38f22c987 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -137,15 +137,21 @@
first))
(defmethod driver.sql/default-schema :clickhouse
- [_driver database]
+ [driver database]
;; ClickHouse opens a database where other engines have a default schema, so an unqualified reference resolves to
;; the one this connection opened rather than to anything the driver could name on its own. `:db` is the older
- ;; spelling of `:dbname`, and both are still in the wild.
+ ;; spelling of `:dbname`, and only `:dbname` reaches the JDBC URL, so details naming a database answer for
+ ;; themselves; details naming none connect anyway, and the server reports where they landed.
(let [details (:details database)]
(or (first-db-name (:dbname details))
(first-db-name (:db details))
- ;; Details carrying neither still connect, to the database `default-connection-details` names.
- (first-db-name (:dbname default-connection-details)))))
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver database nil
+ (fn [^java.sql.Connection conn]
+ (with-open [stmt (.createStatement conn)
+ rset (.executeQuery stmt "SELECT currentDatabase()")]
+ (when (.next rset)
+ (.getString rset 1))))))))
(defmethod sql-jdbc.conn/connection-details->spec :clickhouse
[_ details]
diff --git a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
index 5e4b978ee20..0f2fabfc790 100644
--- a/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
+++ b/modules/drivers/clickhouse/test/metabase/driver/clickhouse_test.clj
@@ -9,6 +9,7 @@
[metabase.driver.clickhouse :as clickhouse]
[metabase.driver.clickhouse-qp :as clickhouse-qp]
[metabase.driver.clickhouse-version :as clickhouse-version]
+ [metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
@@ -33,6 +34,24 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :clickhouse
+ (let [base-details (:details (mt/db))
+ db-name (or (:dbname base-details) (:db base-details))]
+ (testing "database named by the older `db` spelling, which never reaches the JDBC URL"
+ (is (= db-name
+ (driver.sql/default-schema :clickhouse (mt/db)))))
+ (testing "database configured in the connection details"
+ (let [details (assoc base-details :dbname db-name)]
+ (mt/with-temp [:model/Database database {:engine :clickhouse, :details details}]
+ (is (= db-name
+ (driver.sql/default-schema :clickhouse database))))))
+ (testing "details naming no database: the server reports the one the connection opened"
+ (mt/with-temp [:model/Database database {:engine :clickhouse
+ :details (dissoc base-details :db :dbname)}]
+ (is (= "default"
+ (driver.sql/default-schema :clickhouse database))))))))
+
;; the mt/with-dynamic-redefs macro was renamed to mt/with-dynamic-fn-redefs for 0.53+
;; as 0.52 is still tested by CI we will check which macro is defined and use that
(defmacro with-dynamic-redefs [bindings & body]
diff --git a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
index 05f44f2da66..8f17374d769 100644
--- a/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
+++ b/modules/drivers/redshift/test/metabase/driver/redshift_test.clj
@@ -6,6 +6,7 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.driver.redshift :as redshift]
+ [metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
@@ -38,6 +39,18 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :redshift
+ (testing "default"
+ (is (= "public"
+ (driver.sql/default-schema :redshift (mt/db)))))
+ (testing "schema configured in the JDBC additional options"
+ (let [schema (redshift.tx/unique-session-schema)
+ details (assoc (:details (mt/db)) :additional-options (str "currentSchema=" schema))]
+ (mt/with-temp [:model/Database database {:engine :redshift, :details details}]
+ (is (= schema
+ (driver.sql/default-schema :redshift database))))))))
+
(use-fixtures :once (fixtures/initialize :plugins))
(use-fixtures :once (fixtures/initialize :db))
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index c41239b613a..14c8fd5bc43 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -120,8 +120,14 @@
:sunday)
(defmethod driver.sql/default-schema :snowflake
- [_driver _database]
- "PUBLIC")
+ [driver database]
+ ;; Details naming no schema leave the session without one. Metabase has always read an unqualified reference as
+ ;; PUBLIC in that case, so keep doing so rather than resolving it to nothing.
+ (or (sql-jdbc.execute/do-with-connection-with-options
+ driver database nil
+ (fn [^Connection conn]
+ (not-empty (.getSchema conn))))
+ "PUBLIC"))
(defmethod driver/temp-table-name :snowflake
[_driver]
diff --git a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
index 1850c48beb6..30943230709 100644
--- a/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
+++ b/modules/drivers/snowflake/test/metabase/driver/snowflake_test.clj
@@ -14,6 +14,7 @@
[medley.core :as m]
[metabase.driver :as driver]
[metabase.driver.snowflake :as driver.snowflake]
+ [metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
@@ -63,6 +64,17 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :snowflake
+ (testing "details naming no schema fall back to PUBLIC, which is how Metabase has always read them"
+ (is (= "PUBLIC"
+ (driver.sql/default-schema :snowflake (mt/db)))))
+ (testing "schema configured in the JDBC additional options"
+ (let [details (assoc (:details (mt/db)) :additional-options "schema=INFORMATION_SCHEMA")]
+ (mt/with-temp [:model/Database database {:engine :snowflake, :details details}]
+ (is (= "INFORMATION_SCHEMA"
+ (driver.sql/default-schema :snowflake database))))))))
+
(deftest ^:parallel connection-hosts-test
(are [details expected] (= expected (driver/connection-hosts :snowflake details))
{:account "xy12345.us-east-2.aws"}
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index e7b611c08e6..2453be72739 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -112,10 +112,6 @@
[_]
:sunday)
-(defmethod driver.sql/default-schema :sqlserver
- [_driver _database]
- "dbo")
-
(defn- quote-schema [s] (sql.u/quote-name :sqlserver :schema s))
(defn- quote-field [s] (sql.u/quote-name :sqlserver :field s))
diff --git a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
index a698b407879..b5ba8343159 100644
--- a/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
+++ b/modules/drivers/sqlserver/test/metabase/driver/sqlserver_test.clj
@@ -38,6 +38,11 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :sqlserver
+ (is (= "dbo"
+ (driver.sql/default-schema :sqlserver (mt/db))))))
+
(deftest ^:parallel hour-bucketing-time-without-database-type-test
(testing (str "Hour bucketing on a TIME-typed expression without `:database-type` (as happens for "
"fields referenced by name from a source query, #75193) should use TIMEFROMPARTS and "
diff --git a/resources/migrations/065/20260918_database_default_schema.yaml b/resources/migrations/065/20260918_database_default_schema.yaml
new file mode 100644
index 00000000000..8bfdf2417c6
--- /dev/null
+++ b/resources/migrations/065/20260918_database_default_schema.yaml
@@ -0,0 +1,49 @@
+## Persists the schema used for unqualified table references so callers do not need to recalculate it.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.2026-09-18T00:00:06
+ author: ranquild
+ comment: Add default_schema to metabase_database
+ changes:
+ - addColumn:
+ tableName: metabase_database
+ columns:
+ - column:
+ name: default_schema
+ type: varchar(254)
+ remarks: Schema used to resolve unqualified table references; null for databases without schemas
+ constraints:
+ nullable: true
+ rollback:
+ - dropColumn:
+ tableName: metabase_database
+ columnName: default_schema
+
+ - changeSet:
+ id: v65.2026-09-18T00:00:07
+ author: ranquild
+ comment: Backfill metabase_database.default_schema with safe static defaults until the next metadata sync
+ changes:
+ - sql:
+ sql: >-
+ UPDATE metabase_database
+ SET default_schema = CASE
+ WHEN engine IN (
+ 'athena',
+ 'databricks',
+ 'druid-jdbc',
+ 'oracle',
+ 'postgres',
+ 'presto-jdbc',
+ 'redshift',
+ 'sparksql',
+ 'starburst',
+ 'vertica'
+ ) THEN 'public'
+ WHEN engine IN ('h2', 'snowflake') THEN 'PUBLIC'
+ WHEN engine = 'sqlserver' THEN 'dbo'
+ WHEN engine = 'clickhouse' THEN 'default'
+ ELSE NULL
+ END;
+ rollback: # nothing to do; rolling back the preceding changeset drops the column
diff --git a/resources/migrations/065/20260918_transform_test_run.yaml b/resources/migrations/065/20260918_transform_test_run.yaml
new file mode 100644
index 00000000000..841540d5112
--- /dev/null
+++ b/resources/migrations/065/20260918_transform_test_run.yaml
@@ -0,0 +1,120 @@
+## Adds transform_test_run: the execution history for transform tests.
+
+databaseChangeLog:
+ - changeSet:
+ id: v65.2026-09-18T00:00:00
+ author: ranquild
+ comment: Add transform_test_run
+ changes:
+ - createTable:
+ tableName: transform_test_run
+ remarks: An execution of a transform test and its outcome
+ columns:
+ - column:
+ name: id
+ type: int
+ remarks: Unique ID
+ autoIncrement: true
+ constraints:
+ primaryKey: true
+ nullable: false
+ - column:
+ name: transform_test_id
+ type: int
+ remarks: The transform test that was run
+ constraints:
+ nullable: true
+ - column:
+ name: initiated_by
+ type: int
+ remarks: ID of the user who initiated the transform test run
+ constraints:
+ nullable: true
+ - column:
+ name: start_time
+ type: ${timestamp_type}
+ remarks: When the transform test run started
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+ - column:
+ name: end_time
+ type: ${timestamp_type}
+ remarks: When the transform test run completed; null while it is running
+ - column:
+ name: status
+ type: varchar(50)
+ remarks: Current status of the transform test run (started, passed, failed, error, or timeout)
+ constraints:
+ nullable: false
+ - column:
+ name: last_heartbeat
+ type: ${timestamp_type}
+ remarks: Heartbeat stamped while the run executes; a stale value means the owning process died
+ defaultValueComputed: current_timestamp
+ constraints:
+ nullable: false
+
+ - changeSet:
+ id: v65.2026-09-18T00:00:01
+ author: ranquild
+ comment: Index transform_test_run.transform_test_id
+ changes:
+ - createIndex:
+ tableName: transform_test_run
+ indexName: idx_transform_test_run_transform_test_id
+ columns:
+ - column:
+ name: transform_test_id
+
+ - changeSet:
+ id: v65.2026-09-18T00:00:02
+ author: ranquild
+ comment: Add FK constraint on transform_test_run.transform_test_id; SET NULL preserves historical runs when a test is deleted
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: transform_test_run
+ baseColumnNames: transform_test_id
+ referencedTableName: transform_test
+ referencedColumnNames: id
+ constraintName: fk_transform_test_run_transform_test_id
+ onDelete: SET NULL
+
+ - changeSet:
+ id: v65.2026-09-18T00:00:03
+ author: ranquild
+ comment: Index active transform test runs for heartbeat-based orphan detection
+ changes:
+ - createIndex:
+ tableName: transform_test_run
+ indexName: idx_transform_test_run_active_runs
+ columns:
+ - column:
+ name: status
+ - column:
+ name: last_heartbeat
+
+ - changeSet:
+ id: v65.2026-09-18T00:00:04
+ author: ranquild
+ comment: Index transform_test_run.initiated_by
+ changes:
+ - createIndex:
+ tableName: transform_test_run
+ indexName: idx_transform_test_run_initiated_by
+ columns:
+ - column:
+ name: initiated_by
+
+ - changeSet:
+ id: v65.2026-09-18T00:00:05
+ author: ranquild
+ comment: Add FK constraint on transform_test_run.initiated_by; SET NULL preserves historical runs when a user is deleted
+ changes:
+ - addForeignKeyConstraint:
+ baseTableName: transform_test_run
+ baseColumnNames: initiated_by
+ referencedTableName: core_user
+ referencedColumnNames: id
+ constraintName: fk_transform_test_run_initiated_by
+ onDelete: SET NULL
diff --git a/src/metabase/audit_app/events/audit_log.clj b/src/metabase/audit_app/events/audit_log.clj
index f8060fbb8b5..c3dcc407225 100644
--- a/src/metabase/audit_app/events/audit_log.clj
+++ b/src/metabase/audit_app/events/audit_log.clj
@@ -334,6 +334,11 @@
(events/derive! :event/transform-run-start ::transform-event)
(events/derive! :event/transform-run-canceled ::transform-event)
(events/derive! :event/transform-run-timeout ::transform-event)
+(events/derive! :event/transform-test-create ::transform-event)
+(events/derive! :event/transform-test-update ::transform-event)
+(events/derive! :event/transform-test-delete ::transform-event)
+(events/derive! :event/transform-test-run-start ::transform-event)
+(events/derive! :event/transform-test-run-timeout ::transform-event)
(events/derive! :event/transform-inspect-discover ::transform-event)
(events/derive! :event/transform-inspect-lens ::transform-event)
diff --git a/src/metabase/cmd/copy.clj b/src/metabase/cmd/copy.clj
index 2b9e416f98b..38dfcd4affa 100644
--- a/src/metabase/cmd/copy.clj
+++ b/src/metabase/cmd/copy.clj
@@ -161,10 +161,6 @@
:model/TransformTransformTag
:model/TransformJob
:model/TransformJobTransformTag
- ;; By table name: migrations create it on every edition, but its model is EE-only.
- :transform_test
- ;; Serialization never exports run history; a whole-instance move keeps it.
- ;; A run still in flight at dump time arrives marked running, and the transform timeout job reaps it.
:model/TransformJobRun
:model/TransformRun
:model/TransformRunCancelation
@@ -175,7 +171,9 @@
:model/Sandbox
:model/Tenant
:model/ConnectionImpersonation
- :model/CustomVizPlugin])))
+ :model/CustomVizPlugin
+ :model/TransformTest
+ :model/TransformTestRun])))
(defn- objects->columns+values
"Given a sequence of objects/rows fetched from the H2 DB, return a the `columns` that should be used in the `INSERT`
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj
index b148998da22..5cb0d9e23b7 100644
--- a/src/metabase/driver/h2.clj
+++ b/src/metabase/driver/h2.clj
@@ -11,7 +11,6 @@
[metabase.driver.connection :as driver.conn]
[metabase.driver.h2.actions :as h2.actions]
[metabase.driver.settings :as driver.settings]
- [metabase.driver.sql :as sql]
[metabase.driver.sql-jdbc :as sql-jdbc]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.connection.ssh-tunnel :as ssh]
@@ -735,10 +734,6 @@
[_ name-str]
(u/upper-case-en name-str))
-(defmethod sql/default-schema :h2
- [_driver _database]
- "PUBLIC")
-
(defmethod driver/temp-table-name :h2
[_driver]
(str "MB_TEST_" (u/upper-case-en (str/replace (str (random-uuid)) "-" ""))))
diff --git a/src/metabase/driver/sql/normalize.clj b/src/metabase/driver/sql/normalize.clj
index ad43a8085a8..92177694df6 100644
--- a/src/metabase/driver/sql/normalize.clj
+++ b/src/metabase/driver/sql/normalize.clj
@@ -40,11 +40,10 @@
error))
(defmulti default-schema
- "The schema an unqualified table reference resolves to in `database`, or nil when the driver has none.
+ "The schema an unqualified table reference resolves to for `database`, or nil when the driver has none.
- `database` may be nil when the caller has none in hand, and a driver whose default schema is a property of the
- connection rather than of the driver — ClickHouse opens a database where other engines have a default schema —
- answers nil for it.
+ Implementations may connect to the database; callers should use the value persisted by metadata sync instead of
+ invoking this method directly.
Drivers that support any of the `:transforms/...` features must implement this method."
{:added "0.57.0" :arglists '([driver database])}
@@ -55,6 +54,10 @@
[_driver _database]
"public")
+(defmethod default-schema :default
+ [_driver _database]
+ nil)
+
(defmulti reserved-literal
"Checks whether a particular name is actually a literal value in a given sql dialect.
diff --git a/src/metabase/driver/sql_jdbc.clj b/src/metabase/driver/sql_jdbc.clj
index 09f12bd5340..0ae821dfc76 100644
--- a/src/metabase/driver/sql_jdbc.clj
+++ b/src/metabase/driver/sql_jdbc.clj
@@ -1,6 +1,6 @@
(ns metabase.driver.sql-jdbc
"Shared code for drivers for SQL databases using their respective JDBC drivers under the hood."
- (:refer-clojure :exclude [mapv select-keys])
+ (:refer-clojure :exclude [mapv not-empty select-keys])
(:require
[clojure.core.memoize :as memoize]
[clojure.java.jdbc :as jdbc]
@@ -22,7 +22,7 @@
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [mapv select-keys]]
+ [metabase.util.performance :refer [mapv not-empty select-keys]]
[next.jdbc])
(:import
(java.sql Connection SQLException SQLTimeoutException)))
@@ -113,6 +113,13 @@
[driver database]
(sql-jdbc.sync/dbms-version driver (sql-jdbc.conn/db->pooled-connection-spec database)))
+(defmethod driver.sql/default-schema :sql-jdbc
+ [driver database]
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver database nil
+ (fn [^Connection conn]
+ (not-empty (.getSchema conn)))))
+
(defmethod driver/describe-database* :sql-jdbc
[driver database]
(sql-jdbc.sync/describe-database driver database))
diff --git a/src/metabase/driver/sqlite.clj b/src/metabase/driver/sqlite.clj
index 804aba48eb0..4afd29a560a 100644
--- a/src/metabase/driver/sqlite.clj
+++ b/src/metabase/driver/sqlite.clj
@@ -482,6 +482,10 @@
[_driver _database]
"UTC")
+(defmethod driver.sql/default-schema :sqlite
+ [_driver _database]
+ nil)
+
(defmethod sql-jdbc.sync/active-tables :sqlite
[& args]
(apply sql-jdbc.sync/post-filtered-active-tables args))
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index f87860443f9..88daf475d98 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -1105,6 +1105,7 @@
[:name {:optional true} [:maybe :string]]
[:description {:optional true} [:maybe :string]]
[:timezone {:optional true} [:maybe :string]]
+ [:default-schema {:optional true} [:maybe :string]]
[:write-data-details {:optional true} [:maybe ::lib.schema.common/database-details]]
[:admin-details {:optional true} [:maybe ::lib.schema.common/database-details]]
[:router-database-id {:optional true} [:maybe ::lib.schema.id/database]]
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index 1532d00e0af..84eb0424e5b 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -170,7 +170,8 @@
#_resolved-query clojure.lang.IPersistentMap]
[query-type model parsed-args honeysql]
(merge (next-method query-type model parsed-args honeysql)
- {:select [:id :engine :name :dbms_version :settings :is_audit :is_attached_dwh :details :write_data_details :admin_details :timezone :router_database_id]}))
+ {:select [:id :engine :name :dbms_version :settings :is_audit :is_attached_dwh :details :write_data_details
+ :admin_details :timezone :default_schema :router_database_id]}))
(t2/define-after-select :metadata/database
[database]
diff --git a/src/metabase/models/resolution.clj b/src/metabase/models/resolution.clj
index 87a371e72ef..aeb22d315d1 100644
--- a/src/metabase/models/resolution.clj
+++ b/src/metabase/models/resolution.clj
@@ -149,6 +149,7 @@
:model/TransformRunCancelation metabase.transforms.models.transform-run-cancelation
:model/TransformTag metabase.transforms.models.transform-tag
:model/TransformTest metabase-enterprise.transform-testing.models
+ :model/TransformTestRun metabase-enterprise.transform-testing.models.transform-test-run
:model/TransformTransformTag metabase.transforms.models.transform-transform-tag
:model/Undo metabase-enterprise.action-v2.models.undo
:model/SourceDimensionDaily metabase.usage-metadata.models.source-dimension-daily
diff --git a/src/metabase/revisions/api.clj b/src/metabase/revisions/api.clj
index 9423358746c..99163ab0ab1 100644
--- a/src/metabase/revisions/api.clj
+++ b/src/metabase/revisions/api.clj
@@ -4,6 +4,7 @@
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
[metabase.collections.models.collection :as collection]
+ [metabase.config.core :as config]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.models.interface :as mi]
@@ -17,12 +18,13 @@
[toucan2.model :as t2.model]))
(def ^:private entity->model
- {"card" :model/Card
- "dashboard" :model/Dashboard
- "document" :model/Document
- "measure" :model/Measure
- "segment" :model/Segment
- "transform" :model/Transform})
+ (cond-> {"card" :model/Card
+ "dashboard" :model/Dashboard
+ "document" :model/Document
+ "measure" :model/Measure
+ "segment" :model/Segment
+ "transform" :model/Transform}
+ config/ee-available? (assoc "transform-test" :model/TransformTest)))
(def ^:private Entity
"Schema for a valid revisionable entity name."
@@ -124,7 +126,7 @@
;; TODO -- we should be using something like `api/read-check` for this, but unfortunately the impl for Cards
;; doesn't actually check important stuff like this.
(query-perms/check-run-permissions-for-query (dissoc (get-in revision [:object :dataset_query]) :query-permissions/perms)))
- (when (= model :model/Transform)
+ (when (contains? #{:model/Transform :model/TransformTest} model)
(api/check-403 (mi/can-write? (merge instance (:object revision)))))
;; for Segments and Measures `table_id` is re-derived from `definition` on update, so when the restored definition
;; specifies a source table, check write perms against that table rather than the revision's stored `table_id`
diff --git a/src/metabase/revisions/events.clj b/src/metabase/revisions/events.clj
index de82c19f63a..72860dc45e0 100644
--- a/src/metabase/revisions/events.clj
+++ b/src/metabase/revisions/events.clj
@@ -53,6 +53,14 @@
[topic event]
(push-revision! :model/Transform event {:is-creation? (= topic :event/transform-create)}))
+(events/derive! ::transform-test-event ::event)
+(events/derive! :event/transform-test-create ::transform-test-event)
+(events/derive! :event/transform-test-update ::transform-test-event)
+
+(methodical/defmethod events/publish-event! ::transform-test-event
+ [topic event]
+ (push-revision! :model/TransformTest event {:is-creation? (= topic :event/transform-test-create)}))
+
(events/derive! ::segment-event ::event)
(events/derive! :event/segment-create ::segment-event)
(events/derive! :event/segment-update ::segment-event)
diff --git a/src/metabase/sql_tools/common.clj b/src/metabase/sql_tools/common.clj
index 345a4edfbad..00cde57922a 100644
--- a/src/metabase/sql_tools/common.clj
+++ b/src/metabase/sql_tools/common.clj
@@ -47,7 +47,7 @@
An unqualified reference resolves to `database`'s default schema, so the database it was parsed against decides."
[driver database tables transforms {search-table :table raw-schema :schema}]
(let [search-schema (or raw-schema
- (sql.normalize/default-schema driver database))
+ (:default-schema database))
normalize (partial normalize-name driver)
matches? (fn [db-table db-schema]
(and (= (normalize search-table) (normalize db-table))
diff --git a/src/metabase/sql_tools/sqlglot/core.clj b/src/metabase/sql_tools/sqlglot/core.clj
index 00521757e0a..20d82b9be38 100644
--- a/src/metabase/sql_tools/sqlglot/core.clj
+++ b/src/metabase/sql_tools/sqlglot/core.clj
@@ -52,7 +52,7 @@
[driver query]
(try
(let [sql (lib/raw-native-query query)
- default-schema (sql.normalize/default-schema driver (lib.metadata/database query))
+ default-schema (:default-schema (lib.metadata/database query))
query-tables (sql-parsing/referenced-tables (driver->dialect driver) sql)
specs (map (fn [[_catalog table-schema table]]
(sql-tools.common/normalize-table-spec
diff --git a/src/metabase/sync/sync_metadata.clj b/src/metabase/sync/sync_metadata.clj
index 11721101d79..30d9a165148 100644
--- a/src/metabase/sync/sync_metadata.clj
+++ b/src/metabase/sync/sync_metadata.clj
@@ -2,14 +2,16 @@
"Logic responsible for syncing the metadata for an entire database.
Delegates to different subtasks:
- 1. Sync tables (`metabase.sync.sync-metadata.tables`)
- 2. Sync fields (`metabase.sync.sync-metadata.fields`)
- 3. Sync FKs (`metabase.sync.sync-metadata.fks`)
- 4. Sync Metabase Metadata table (`metabase.sync.sync-metadata.metabase-metadata`)"
+ 1. Sync database properties (DBMS version, timezone, and default schema)
+ 2. Sync tables (`metabase.sync.sync-metadata.tables`)
+ 3. Sync fields (`metabase.sync.sync-metadata.fields`)
+ 4. Sync FKs and indexes
+ 5. Sync Metabase Metadata table (`metabase.sync.sync-metadata.metabase-metadata`)"
(:require
[metabase.sync.fetch-metadata :as fetch-metadata]
[metabase.sync.interface :as i]
[metabase.sync.sync-metadata.dbms-version :as sync-dbms-ver]
+ [metabase.sync.sync-metadata.default-schema :as sync-default-schema]
[metabase.sync.sync-metadata.fields :as sync-fields]
[metabase.sync.sync-metadata.fks :as sync-fks]
[metabase.sync.sync-metadata.indexes :as sync-indexes]
@@ -37,6 +39,9 @@
(defn- sync-timezone-summary [{:keys [timezone-id]}]
(format "Found timezone id %s" timezone-id))
+(defn- sync-default-schema-summary [{:keys [default-schema]}]
+ (format "Found default schema %s" default-schema))
+
(defn- sync-fks-summary [{:keys [total-fks updated-fks total-failed]}]
(format "Total number of foreign keys sync''d %d, %d updated and %d tables failed to update"
total-fks updated-fks total-failed))
@@ -48,6 +53,7 @@
(defn- make-sync-steps [db-metadata]
[(sync-util/create-sync-step "sync-dbms-version" sync-dbms-ver/sync-dbms-version! sync-dbms-version-summary)
(sync-util/create-sync-step "sync-timezone" sync-tz/sync-timezone! sync-timezone-summary)
+ (sync-util/create-sync-step "sync-default-schema" sync-default-schema/sync-default-schema! sync-default-schema-summary)
;; Make sure the relevant table models are up-to-date
(sync-util/create-sync-step "sync-tables" #(sync-tables/sync-tables-and-database! % db-metadata) sync-tables-summary true)
;; Now for each table, sync the fields
diff --git a/src/metabase/sync/sync_metadata/default_schema.clj b/src/metabase/sync/sync_metadata/default_schema.clj
new file mode 100644
index 00000000000..00bb34abdf1
--- /dev/null
+++ b/src/metabase/sync/sync_metadata/default_schema.clj
@@ -0,0 +1,16 @@
+(ns metabase.sync.sync-metadata.default-schema
+ (:require
+ [metabase.driver.sql :as driver.sql]
+ [metabase.driver.util :as driver.u]
+ [metabase.sync.db :as sync.db]
+ [metabase.sync.interface :as i]
+ [metabase.util.malli :as mu]))
+
+(mu/defn sync-default-schema! :- [:map [:default-schema [:maybe :string]]]
+ "Ask the driver which schema unqualified table references use and persist it on `database`."
+ [database :- i/DatabaseInstance]
+ (let [driver (driver.u/database->driver database)
+ default-schema (driver.sql/default-schema driver database)]
+ (when-not (= default-schema (:default_schema database))
+ (sync.db/update-database! (:id database) {:default_schema default-schema}))
+ {:default-schema default-schema}))
diff --git a/src/metabase/sync/util.clj b/src/metabase/sync/util.clj
index ec136121068..7b8d99b7c04 100644
--- a/src/metabase/sync/util.clj
+++ b/src/metabase/sync/util.clj
@@ -541,6 +541,7 @@
[:added-indexes {:optional true} :int]
[:cloud {:optional true} :boolean]
[:created {:optional true} :int]
+ [:default-schema {:optional true} [:maybe :string]]
[:deleted {:optional true} :int]
[:errors {:optional true} :int]
[:failed-fingerprints {:optional true} :int]
diff --git a/src/metabase/task_history/schema.clj b/src/metabase/task_history/schema.clj
index bf00459c416..f4472cada6b 100644
--- a/src/metabase/task_history/schema.clj
+++ b/src/metabase/task_history/schema.clj
@@ -76,6 +76,7 @@
[:map {:closed true}
[:version {:optional true} [:maybe :string]]
[:timezone-id {:optional true} [:maybe :string]]
+ [:default-schema {:optional true} [:maybe :string]]
[:total-tables {:optional true} :int]
[:updated-tables {:optional true} :int]
[:tables-classified {:optional true} :int]
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 0d612f46fc6..e35ab324e33 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -8,7 +8,6 @@
[java-time.api :as t]
[metabase.database-routing.core :as database-routing]
[metabase.driver :as driver]
- [metabase.driver.sql.normalize :as sql.normalize]
[metabase.events.core :as events]
[metabase.indexes.models.table-index :as table-index]
[metabase.indexes.reconcile :as reconcile]
@@ -183,7 +182,7 @@
This handles the case where transforms create tables without explicit schema
but the driver needs a schema to find the table during sync."
[driver database table]
- (when-let [default-schema (try (sql.normalize/default-schema driver database) (catch Exception _ nil))]
+ (when-let [default-schema (:default_schema database)]
(when (driver/table-exists? driver database {:schema default-schema :name (:name table)})
default-schema)))
diff --git a/src/metabase/warehouses/models/database.clj b/src/metabase/warehouses/models/database.clj
index 4b65bf3a9ed..f0ac87eb95b 100644
--- a/src/metabase/warehouses/models/database.clj
+++ b/src/metabase/warehouses/models/database.clj
@@ -694,7 +694,7 @@
:import identity}]
{:copy [:auto_run_queries :cache_field_values_schedule :caveats :dbms_version
:description :engine :is_audit :is_attached_dwh :is_full_sync :is_on_demand :is_sample :is_stub
- :metadata_sync_schedule :name :points_of_interest :provider_name :refingerprint :settings :timezone :uploads_enabled
+ :default_schema :metadata_sync_schedule :name :points_of_interest :provider_name :refingerprint :settings :timezone :uploads_enabled
:uploads_schema_name :uploads_table_prefix]
:skip [;; deprecated field
:cache_ttl]
diff --git a/src/metabase/warehouses/schema.clj b/src/metabase/warehouses/schema.clj
index 3de3b7781dc..63ca5f8ed1d 100644
--- a/src/metabase/warehouses/schema.clj
+++ b/src/metabase/warehouses/schema.clj
@@ -70,6 +70,7 @@
[:metadata_sync_schedule {:optional true} [:maybe :string]]
[:cache_field_values_schedule {:optional true} [:maybe :string]]
[:timezone {:optional true} [:maybe :string]]
+ [:default_schema {:optional true} [:maybe :string]]
[:is_on_demand {:optional true} [:maybe :boolean]]
[:auto_run_queries {:optional true} [:maybe :boolean]]
[:refingerprint {:optional true} [:maybe :boolean]]
diff --git a/test/metabase/cmd/copy_test.clj b/test/metabase/cmd/copy_test.clj
index 198d40ffd52..7c7f0cec9fb 100644
--- a/test/metabase/cmd/copy_test.clj
+++ b/test/metabase/cmd/copy_test.clj
@@ -9,8 +9,15 @@
[metabase.app-db.setup :as mdb.setup]
[metabase.classloader.core :as classloader]
[metabase.cmd.copy :as copy]
+ [metabase.config.core :as config]
[toucan2.core :as t2]))
+(deftest ^:parallel transform-testing-tables-are-ee-only-test
+ (let [entities (set copy/entities)]
+ (doseq [model [:model/TransformTest :model/TransformTestRun]]
+ (is (= config/ee-available? (contains? entities model))))
+ (is (not-any? entities [:transform_test :transform_test_run]))))
+
(deftest ^:parallel sql-for-selecting-instances-from-source-db-test
(is (= "SELECT * FROM metabase_field ORDER BY id ASC"
(#'copy/sql-for-selecting-instances-from-source-db :model/Field)))
@@ -119,8 +126,6 @@
:model/TableIndex
:model/TaskHistory
:model/TaskRun
- ;; copied by table name instead, since its model is EE-only
- :model/TransformTest
:model/Undo
:model/UserKeyValue})
diff --git a/test/metabase/driver/h2_test.clj b/test/metabase/driver/h2_test.clj
index 01a6184331f..a8defb6603d 100644
--- a/test/metabase/driver/h2_test.clj
+++ b/test/metabase/driver/h2_test.clj
@@ -13,6 +13,7 @@
[metabase.driver.h2 :as h2]
[metabase.driver.h2.actions :as h2.actions]
[metabase.driver.settings :as driver.settings]
+ [metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.actions :as sql-jdbc.actions]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql.query-processor :as sql.qp]
@@ -27,6 +28,17 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :h2
+ (testing "default"
+ (is (= "PUBLIC"
+ (driver.sql/default-schema :h2 (mt/db)))))
+ (testing "schema configured in the JDBC URL"
+ (let [details (update (:details (mt/db)) :db str ";SCHEMA=INFORMATION_SCHEMA")]
+ (mt/with-temp [:model/Database database {:engine :h2, :details details}]
+ (is (= "INFORMATION_SCHEMA"
+ (driver.sql/default-schema :h2 database))))))))
+
(deftest ^:parallel connection-hosts-test
(testing "local H2 databases have no network host"
(is (= [] (driver/connection-hosts :h2 {:db "file:./sample.db"})))
diff --git a/test/metabase/driver/mysql_test.clj b/test/metabase/driver/mysql_test.clj
index 6781b5a6c5c..3a86701af71 100644
--- a/test/metabase/driver/mysql_test.clj
+++ b/test/metabase/driver/mysql_test.clj
@@ -17,6 +17,7 @@
[metabase.driver.mysql.actions :as mysql.actions]
[metabase.driver.mysql.ddl :as mysql.ddl]
[metabase.driver.settings :as driver.settings]
+ [metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.actions :as sql-jdbc.actions]
[metabase.driver.sql-jdbc.actions-test :as sql-jdbc.actions-test]
@@ -53,6 +54,10 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :mysql
+ (is (nil? (driver.sql/default-schema :mysql (mt/db))))))
+
(use-fixtures :each (fn [thunk]
;; 1. If sync fails when loading a test dataset, don't swallow the error; throw an Exception so we
;; can debug it. This is much less confusing when trying to fix broken tests.
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index feb0884d86c..9cd292e9875 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -73,6 +73,17 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :postgres
+ (testing "default"
+ (is (= "public"
+ (driver.sql/default-schema :postgres (mt/db)))))
+ (testing "schema configured in the JDBC additional options"
+ (let [details (assoc (:details (mt/db)) :additional-options "currentSchema=information_schema")]
+ (mt/with-temp [:model/Database database {:engine :postgres, :details details}]
+ (is (= "information_schema"
+ (driver.sql/default-schema :postgres database))))))))
+
(use-fixtures :each (fn [thunk]
;; 1. If sync fails when loading a test dataset, don't swallow the error; throw an Exception so we
;; can debug it. This is much less confusing when trying to fix broken tests.
@@ -2551,6 +2562,7 @@
(testing "`final` is allowed as identifier and parsed correctly"
(mt/with-temp [:model/Database db {:engine "postgres"
:name "final"
+ :default_schema "public"
:initial_sync_status "complete"}
:model/Table t {:name "final"
:schema "public"
diff --git a/test/metabase/driver/sqlite_test.clj b/test/metabase/driver/sqlite_test.clj
index 1a33d3b30c1..1403a0236e1 100644
--- a/test/metabase/driver/sqlite_test.clj
+++ b/test/metabase/driver/sqlite_test.clj
@@ -6,6 +6,7 @@
[clojure.java.jdbc :as jdbc]
[clojure.test :refer :all]
[metabase.driver :as driver]
+ [metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql.query-processor :as sql.qp]
@@ -21,6 +22,10 @@
(set! *warn-on-reflection* true)
+(deftest default-schema-test
+ (mt/test-driver :sqlite
+ (is (nil? (driver.sql/default-schema :sqlite (mt/db))))))
+
(deftest ^:parallel timezone-id-test
(mt/test-driver :sqlite
(is (= "UTC"
diff --git a/test/metabase/lib/test_metadata.cljc b/test/metabase/lib/test_metadata.cljc
index ea3b60b00be..37cee52b8ef 100644
--- a/test/metabase/lib/test_metadata.cljc
+++ b/test/metabase/lib/test_metadata.cljc
@@ -4024,6 +4024,7 @@
:is-on-demand false
:options nil
:engine :h2
+ :default-schema "PUBLIC"
:initial-sync-status "complete"
:native-permissions :write
:dbms-version {:flavor "H2", :version "2.1.212 (2022-04-09)", :semantic-version [2 1]}
diff --git a/test/metabase/lib_be/metadata/jvm_test.clj b/test/metabase/lib_be/metadata/jvm_test.clj
index 50621e216ea..c640939450a 100644
--- a/test/metabase/lib_be/metadata/jvm_test.clj
+++ b/test/metabase/lib_be/metadata/jvm_test.clj
@@ -23,7 +23,7 @@
(is (not (me/humanize (mr/explain ::lib.schema.metadata/column field))))))
(deftest ^:parallel fetch-database-test
- (is (=? {:lib/type :metadata/database, :features set?}
+ (is (=? {:lib/type :metadata/database, :features set?, :default-schema "PUBLIC"}
(lib.metadata/database (mt/metadata-provider))))
(testing "Should return nil correctly"
(is (nil? (lib.metadata.protocols/database (lib.metadata.jvm/application-database-metadata-provider Integer/MAX_VALUE))))))
diff --git a/test/metabase/sql_tools/common_test.clj b/test/metabase/sql_tools/common_test.clj
index b4e9d980ef7..9978f0e7630 100644
--- a/test/metabase/sql_tools/common_test.clj
+++ b/test/metabase/sql_tools/common_test.clj
@@ -48,3 +48,15 @@
(is (= #{} (sql-tools.common/table-ids-by-name db-id ["no_such_table"]))))
(testing "no names requested returns empty rather than the whole catalog"
(is (= #{} (sql-tools.common/table-ids-by-name db-id []))))))))
+
+(deftest find-table-or-transform-uses-synced-default-schema-test
+ (let [database {:default-schema "analytics"}
+ tables [{:id 1 :name "orders" :schema "analytics"}
+ {:id 2 :name "orders" :schema "public"}]]
+ (testing "an unqualified reference uses the Database's synced default schema"
+ (is (= {:table 1}
+ (sql-tools.common/find-table-or-transform :h2 database tables [] {:table "orders"}))))
+ (testing "an explicit schema takes precedence"
+ (is (= {:table 2}
+ (sql-tools.common/find-table-or-transform
+ :h2 database tables [] {:table "orders" :schema "public"}))))))
diff --git a/test/metabase/sync/sync_metadata/default_schema_test.clj b/test/metabase/sync/sync_metadata/default_schema_test.clj
new file mode 100644
index 00000000000..adfe9ceeb23
--- /dev/null
+++ b/test/metabase/sync/sync_metadata/default_schema_test.clj
@@ -0,0 +1,22 @@
+(ns ^:mb/driver-tests metabase.sync.sync-metadata.default-schema-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.sync.util-test :as sync.util-test]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
+
+(deftest sync-default-schema-test
+ (mt/test-driver :h2
+ (mt/dataset test-data
+ (let [database (mt/db)]
+ (t2/update! :model/Database (:id database) {:default_schema nil})
+ (let [{:keys [step-info task-history]}
+ (sync.util-test/sync-database! "sync-default-schema" (assoc database :default_schema nil))]
+ (testing "the sync step reports the discovered schema"
+ (is (= {:default-schema "PUBLIC"}
+ (sync.util-test/only-step-keys step-info)))
+ (is (= {:default-schema "PUBLIC"}
+ (:task_details task-history))))
+ (testing "the discovered schema is persisted on the Database"
+ (is (= "PUBLIC"
+ (t2/select-one-fn :default_schema :model/Database (:id database))))))))))
diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj
index 1e45d7cfe11..c91bd6f7d22 100644
--- a/test/metabase/test/util.clj
+++ b/test/metabase/test/util.clj
@@ -414,6 +414,12 @@
:inputs []
:expectations []}))
+ :model/TransformTestRun
+ (fn [_]
+ {:status "started"
+ :start_time (t/instant)
+ :last_heartbeat (t/instant)})
+
:model/TransformRun
(fn [_]
{:status "succeeded"
diff --git a/test/metabase/transforms/test_util.clj b/test/metabase/transforms/test_util.clj
index 67ea47db92d..a1839e25258 100644
--- a/test/metabase/transforms/test_util.clj
+++ b/test/metabase/transforms/test_util.clj
@@ -4,7 +4,6 @@
[clojure.test :refer :all]
[metabase.api.common :as api]
[metabase.driver :as driver]
- [metabase.driver.sql :as driver.sql]
[metabase.driver.util :as driver.u]
[metabase.test :as mt]
[metabase.test.data.interface :as tx]
@@ -68,7 +67,7 @@
;; assume this is just a plain table name
{:type :table, :name target})
(and (nil? (:schema target)) (isa? driver/hierarchy driver/*driver* :sql))
- (assoc :schema (driver.sql/default-schema driver/*driver* (mt/db))))]
+ (assoc :schema (:default_schema (mt/db))))]
(binding [api/*is-superuser?* true
api/*current-user-id* (mt/user->id :crowberto)]
;; Drop the actual table/view from the database
@@ -251,8 +250,7 @@
"Returns the driver's default schema (e.g., 'dbo' for SQL Server) or 'public' as fallback.
Useful for tests that need to create tables with a schema matching transform targets."
[]
- (or (when (get-method driver.sql/default-schema driver/*driver*)
- (driver.sql/default-schema driver/*driver* (mt/db)))
+ (or (:default_schema (mt/db))
(if (= :bigquery-cloud-sdk driver/*driver*)
(t2/select-one-fn :schema :model/Table :db_id (mt/id))
"public")))
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index de79472a41a..bef57ed0788 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -779,7 +779,7 @@
(testing "activate-table-and-mark-computed! resolves nil schema when physical table exists in default schema"
(let [target {:type "table" :schema nil :name "test_nil_schema_fix"}
synced-table (atom nil)]
- (mt/with-temp [:model/Database db {:engine :h2}]
+ (mt/with-temp [:model/Database db {:engine :h2, :default_schema "PUBLIC"}]
(with-redefs [sync/create-table! (fn [database table-map]
(let [created (t2/insert-returning-instance!
:model/Table
diff --git a/test/metabase/transforms_base/ordering_test.clj b/test/metabase/transforms_base/ordering_test.clj
index 36669ab54a1..f10570af996 100644
--- a/test/metabase/transforms_base/ordering_test.clj
+++ b/test/metabase/transforms_base/ordering_test.clj
@@ -1,8 +1,6 @@
(ns ^:mb/driver-tests metabase.transforms-base.ordering-test
(:require
[clojure.test :refer :all]
- [metabase.driver :as driver]
- [metabase.driver.sql :as driver.sql]
[metabase.query-processor.compile :as qp.compile]
[metabase.test :as mt]
[metabase.test.data.sql :as sql.tx]
@@ -11,9 +9,8 @@
[metabase.transforms.execute :as transforms.execute]
[toucan2.core :as t2]))
-(defn- default-schema-or-public [& [fallback-driver]]
- (let [driver (or driver/*driver* fallback-driver)]
- (or (and driver (driver.sql/default-schema driver (mt/db))) "public")))
+(defn- default-schema-or-public []
+ (or (:default_schema (mt/db)) "public"))
(defn- make-transform [query & [name schema]]
(let [name (or name (mt/random-name))
diff --git a/test/metabase/warehouse_schema_rest/api/table_test.clj b/test/metabase/warehouse_schema_rest/api/table_test.clj
index d8d9b43af34..f7b695ae656 100644
--- a/test/metabase/warehouse_schema_rest/api/table_test.clj
+++ b/test/metabase/warehouse_schema_rest/api/table_test.clj
@@ -51,6 +51,7 @@
(defn- db-details []
(merge
(select-keys (mt/db) [:id :entity_id :created_at :updated_at :timezone :creator_id :initial_sync_status :dbms_version
+ :default_schema
:cache_field_values_schedule :metadata_sync_schedule :uploads_enabled :uploads_schema_name
:uploads_table_prefix])
{:engine "h2"
diff --git a/test/metabase/warehouses_rest/api_test.clj b/test/metabase/warehouses_rest/api_test.clj
index 62c3286dcb8..f707aeea57f 100644
--- a/test/metabase/warehouses_rest/api_test.clj
+++ b/test/metabase/warehouses_rest/api_test.clj
@@ -90,7 +90,7 @@
([{driver :engine, :as db}]
(merge
(mt/object-defaults :model/Database)
- (select-keys db [:created_at :id :details :updated_at :timezone :name :dbms_version
+ (select-keys db [:created_at :id :details :updated_at :timezone :name :dbms_version :default_schema
:metadata_sync_schedule :cache_field_values_schedule :uploads_enabled :uploads_schema_name])
{:engine (u/qualified-name (:engine db))
:settings {}