Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add kondo rule keeping Toucan 2 query calls in module db namespaces (#82099)

Alexander Polyankin15106f17fda5
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn
index 80424f7a1db..65998f56f9c 100644
--- a/.clj-kondo/config.edn
+++ b/.clj-kondo/config.edn
@@ -72,6 +72,7 @@
:metabase/namespace-name {:level :warning}
:metabase/no-jsqlparser-imports {:level :warning}
:metabase/prefer-with-dynamic-fn-redefs {:level :warning}
+ :metabase/t2-query-namespace {:level :warning}
:metabase/toucan-model-ns {:level :warning}
:metabase/require-shape-checker {:level :warning}
:metabase/tests-must-live-in-known-modules {:level :warning}
@@ -633,6 +634,45 @@
metabase.lib.schema.mbql-clause/define-tuple-mbql-clause hooks.metabase.lib.schema.mbql-clause/define-mbql-clause
methodical.core/defmethod hooks.metabase.toucan.table-name/lint-defmethod
methodical.macros/defmethod hooks.metabase.toucan.table-name/lint-defmethod
+ metabase.app-db.core/query hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.core/streaming-reducible-query hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.core/select-or-insert! hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.core/update-or-insert! hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.core/current-timestamp-string hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.query/query hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.query/streaming-reducible-query hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.query/select-or-insert! hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.query/update-or-insert! hooks.metabase.toucan.db-ns/lint-query-call
+ metabase.app-db.query/current-timestamp-string hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/count hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/delete! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/exists? hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/insert! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/insert-returning-instance! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/insert-returning-instances! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/insert-returning-pk! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/insert-returning-pks! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/query hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/query-one hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/reducible-query hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/reducible-select hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/reducible-update hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/reducible-update-returning-pks hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/save! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-fn->fn hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-fn->pk hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-fn-reducible hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-fn-set hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-fn-vec hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-one hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-one-fn hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-one-pk hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-pk->fn hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-pks-set hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/select-pks-vec hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/update! hooks.metabase.toucan.db-ns/lint-query-call
+ toucan2.core/update-returning-pks! hooks.metabase.toucan.db-ns/lint-query-call
metabase.lib.test-util.macros/$ids hooks.metabase.test.data/$ids
metabase.lib.test-util.macros/mbql-query hooks.metabase.test.data/mbql-query
metabase.lib.test-util.macros/mbql-5-query hooks.metabase.test.data/mbql-query
@@ -841,7 +881,13 @@
{:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$"
:name general-qp-and-driver-test-namespaces}
{:pattern "(?:^mage.*)"
- :name mage-namespaces}]
+ :name mage-namespaces}
+ ;;
+ ;; the app-db module is the application database layer itself (connections, migrations, encryption), so it gets to
+ ;; run Toucan 2 queries anywhere.
+ ;;
+ {:pattern "^metabase\\.app-db\\..*"
+ :name app-db-namespaces}]
:config-in-ns
{test-namespaces
@@ -851,6 +897,7 @@
:private-call {:level :off}
:metabase/defsetting-must-specify-export {:level :off}
:metabase/defsetting-namespace {:level :off}
+ :metabase/t2-query-namespace {:level :off}
:metabase/test-helpers-use-non-thread-safe-functions {:level :warning} ; more config in `.clj-kondo/config/test-helpers/config.edn`
:discouraged-var
@@ -858,6 +905,9 @@
metabase.lib.convert/->legacy-MBQL {:level :off}
metabase.lib.core/->legacy-MBQL {:level :off}}}}
+ app-db-namespaces
+ {:linters {:metabase/t2-query-namespace {:level :off}}}
+
;; .cljc tests run in both Clojure and ClojureScript — `with-dynamic-fn-redefs` is JVM-only,
;; so leave these alone.
metabase.lib.aggregation-test
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 0749f5ba677..dfab7040c23 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -61,6 +61,7 @@
:metabase/defsetting-must-specify-export 1
:metabase/defsetting-namespace 1
:metabase/prefer-with-dynamic-fn-redefs 3
+ :metabase/t2-query-namespace 2
:missing-docstring 1
:private-call 1
:redundant-ignore 1
diff --git a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
new file mode 100644
index 00000000000..db2296c7a69
--- /dev/null
+++ b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj
@@ -0,0 +1,35 @@
+(ns hooks.metabase.toucan.db-ns
+ "Lint that application database query calls -- Toucan 2's `t2/select`, `t2/query`, `t2/insert!`, `t2/update!`,
+ `t2/delete!` and friends, plus the `metabase.app-db.core` wrappers around them such as `mdb/query` and
+ `mdb/update-or-insert!` -- live in a module's `db` namespace, i.e. `metabase[-enterprise].<module>.db` (or `metabase.driver.<driver>.db` for
+ driver modules). Every other namespace in a module goes through those functions instead of talking to the
+ application database directly.
+
+ Registered as an `:analyze-call` hook on each of those functions in `.clj-kondo/config.edn`. The hook
+ returns its input unchanged so Kondo's normal analysis of the call (arity, var usage) still runs."
+ (:require
+ [clj-kondo.hooks-api :as hooks]))
+
+(defn- db-namespace? [ns-sym]
+ (boolean (re-matches #"^metabase(?:-enterprise)?\.(?:driver\.)?[^.]+\.db$" (name ns-sym))))
+
+(defn- test-file?
+ "Whether `filename` is in a test source tree. Test namespaces are exempt through the `test-namespaces` group in
+ `config.edn`, but helper namespaces like `metabase.sso.test-helpers` don't match that group's pattern."
+ [filename]
+ (boolean (and filename (re-find #"(?:^|/)test/" filename))))
+
+(defn lint-query-call
+ "Register a `:metabase/t2-query-namespace` finding when a Toucan 2 query call appears outside a `<module>.db`
+ namespace."
+ [{:keys [node ns filename] :as input}]
+ (when (and ns
+ (not (db-namespace? ns))
+ (not (test-file? filename)))
+ (let [fn-node (first (:children node))]
+ (hooks/reg-finding!
+ (assoc (meta fn-node)
+ :message (format "Application database query calls like `%s` must live in metabase[-enterprise].<module>.db (or metabase.driver.<driver>.db) namespaces"
+ (hooks/sexpr fn-node))
+ :type :metabase/t2-query-namespace))))
+ input)
diff --git a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj
new file mode 100644
index 00000000000..2109ad98d1d
--- /dev/null
+++ b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj
@@ -0,0 +1,50 @@
+(ns hooks.metabase.toucan.db-ns-test
+ (:require
+ [clj-kondo.hooks-api :as hooks]
+ [clj-kondo.impl.utils]
+ [clojure.test :refer :all]
+ [hooks.metabase.toucan.db-ns :as toucan.db-ns]))
+
+(defn- lint-query-call [form ns-sym & [filename]]
+ (binding [clj-kondo.impl.utils/*ctx* {:config {:linters {:metabase/t2-query-namespace {:level :warning}}}
+ :ignores (atom nil)
+ :findings (atom [])
+ :namespaces (atom {})}]
+ (let [input {:node (hooks/parse-string (pr-str form))
+ :ns ns-sym
+ :filename (or filename "src/metabase/foo/bar.clj")}
+ output (toucan.db-ns/lint-query-call input)]
+ (is (identical? (:node input) (:node output))
+ "the hook must return the node unchanged so Kondo's normal analysis still runs")
+ @(:findings clj-kondo.impl.utils/*ctx*))))
+
+(deftest ^:parallel t2-query-must-live-in-db-namespace-test
+ (testing "a query call outside a db namespace is flagged"
+ (is (=? [{:type :metabase/t2-query-namespace
+ :message #".*`t2/select-one`.*"}]
+ (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase.queries.models.card))))
+ (testing "metabase.<module>.db is allowed"
+ (is (empty? (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase.queries.db))))
+ (testing "metabase-enterprise.<module>.db is allowed"
+ (is (empty? (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase-enterprise.sandbox.db))))
+ (testing "metabase.driver.<driver>.db is allowed for driver modules"
+ (is (empty? (lint-query-call '(t2/select-one :model/Database :id 1) 'metabase.driver.bigquery-cloud-sdk.db))))
+ (testing "a nested db namespace is not a module db namespace"
+ (is (=? [{:type :metabase/t2-query-namespace}]
+ (lint-query-call '(t2/query {:select [:*]}) 'metabase.queries.models.db))))
+ (testing "a namespace merely ending in db is not a module db namespace"
+ (is (=? [{:type :metabase/t2-query-namespace}]
+ (lint-query-call '(t2/delete! :model/Card :id 1) 'metabase.queries.dashboards-db))))
+ (testing "files in a test source tree are exempt even when the namespace name doesn't look like a test"
+ (is (empty? (lint-query-call '(t2/update! :model/User 1 {:sso_source nil}) 'metabase.sso.test-helpers
+ "test/metabase/sso/test_helpers.clj")))
+ (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase-enterprise.remote-sync.test-helpers
+ "/repo/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj"))))
+ (testing "app-db wrappers around Toucan 2 are reported the same way"
+ (is (=? [{:type :metabase/t2-query-namespace
+ :message #".*`mdb/update-or-insert!`.*"}]
+ (lint-query-call '(mdb/update-or-insert! :model/Foo {:id 1}) 'metabase.foo.models.foo))))
+ (testing "fully-qualified calls are reported by their written name"
+ (is (=? [{:type :metabase/t2-query-namespace
+ :message #".*`toucan2.core/insert!`.*"}]
+ (lint-query-call '(toucan2.core/insert! :model/Card {}) 'metabase.queries.models.card)))))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index 64ac847cb58..c62cf4faa9f 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -608,6 +608,40 @@
[entity-type entity-id]
(t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id entity-id))
+(defn mark-dependency-status-stale!
+ "Mark the DependencyStatus of `entity-type` `entity-id` as stale for dependency recalculation, creating it if it
+ doesn't exist. Resets retry state so previously-failed entities get a fresh chance."
+ [entity-type entity-id]
+ (mdb/update-or-insert!
+ :model/DependencyStatus
+ {:entity_type entity-type :entity_id entity-id}
+ (fn [existing]
+ (if existing
+ {:stale true :fail_count 0 :next_retry_at nil :terminal false}
+ {:stale true :dependency_analysis_version 0}))))
+
+(defn upsert-dependency-status!
+ "Upsert the DependencyStatus of `entity-type` `entity-id`, setting stale=false, `dependency_analysis_version` to
+ `current-version`, and clearing any failure state."
+ [entity-type entity-id current-version]
+ (mdb/update-or-insert!
+ :model/DependencyStatus
+ {:entity_type entity-type :entity_id entity-id}
+ (fn [_existing]
+ {:dependency_analysis_version current-version
+ :stale false
+ :fail_count 0
+ :next_retry_at nil})))
+
+(defn record-dependency-status-failure!
+ "Upsert the DependencyStatus of `entity-type` `entity-id`, creating it if needed. `update-fn` receives the
+ existing DependencyStatus row (or nil if none exists) and must return the columns to set."
+ [entity-type entity-id update-fn]
+ (mdb/update-or-insert!
+ :model/DependencyStatus
+ {:entity_type entity-type :entity_id entity-id}
+ update-fn))
+
(defn pending-retry-exists?
"Whether a non-terminal DependencyStatus is waiting for a retry."
[]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
index 8f28846bfda..0513a1660d2 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj
@@ -4,7 +4,6 @@
[metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.dependency-types :as deps.dependency-types]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
- [metabase.app-db.core :as app-db]
[metabase.models.interface :as mi]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -19,30 +18,17 @@
(defn mark-stale!
"Mark entities of `entity-type` with ids in `entity-ids` as stale for dependency recalculation.
Creates entries if they don't exist, or sets stale=true if they do.
- Resets retry state so previously-failed entities get a fresh chance.
- Uses [[app-db/update-or-insert!]] for cross-database atomicity."
+ Resets retry state so previously-failed entities get a fresh chance."
[entity-type entity-ids]
(doseq [id entity-ids]
- (app-db/update-or-insert!
- :model/DependencyStatus
- {:entity_type entity-type :entity_id id}
- (fn [existing]
- (if existing
- {:stale true :fail_count 0 :next_retry_at nil :terminal false}
- {:stale true :dependency_analysis_version 0})))))
+ (dependencies.db/mark-dependency-status-stale! entity-type id)))
(defn upsert-status!
"Upsert a dependency_status entry, setting stale=false, version to current,
- and clearing any failure state. Uses [[app-db/update-or-insert!]] for cross-database atomicity."
+ and clearing any failure state."
[entity-type entity-id]
- (app-db/update-or-insert!
- :model/DependencyStatus
- {:entity_type entity-type :entity_id entity-id}
- (fn [_existing]
- {:dependency_analysis_version models.dependency/current-dependency-analysis-version
- :stale false
- :fail_count 0
- :next_retry_at nil})))
+ (dependencies.db/upsert-dependency-status!
+ entity-type entity-id models.dependency/current-dependency-analysis-version))
(defmulti hydrate-for-deps
"Hydrate a batch of instances with data needed for dependency calculation.
@@ -87,12 +73,10 @@
Increments fail_count and sets next_retry_at based on exponential backoff.
If max retries exceeded, marks the entity as terminal.
Creates the entry if it doesn't exist, since entities with no row yet are exactly the ones
- [[instances-for-dependency-calculation]] picks up first.
- Uses [[app-db/update-or-insert!]] for cross-database atomicity."
+ [[instances-for-dependency-calculation]] picks up first."
[entity-type entity-id max-retries delay-minutes]
- (app-db/update-or-insert!
- :model/DependencyStatus
- {:entity_type entity-type :entity_id entity-id}
+ (dependencies.db/record-dependency-status-failure!
+ entity-type entity-id
(fn [existing]
;; An inserted row takes the column defaults for `stale` (false) and `dependency_analysis_version` (0). 0 is
;; below `current-dependency-analysis-version`, which is what keeps the entity eligible for the retry once
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
index ef42ea87ba3..d0269500662 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the sandbox module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module only touches `toucan2.core` for model definitions."
(:require
+ [metabase.app-db.core :as mdb]
[toucan2.core :as t2]))
(defn sandbox
@@ -54,6 +55,23 @@
(when db-id [:= :t.db_id db-id])
(when excluded-db-id [:not [:= :t.db_id excluded-db-id]])]}))
+(defn candidate-sandboxes-for-groups-and-databases
+ "The `:id`, `:group_id`, `:table_id`, `:db_id`, and `:schema` of the Sandboxes of the groups with `group-ids` on
+ Tables of the Databases with `db-ids`."
+ [group-ids db-ids]
+ (mdb/query
+ {:select [[:sandboxes.id :id]
+ [:sandboxes.group_id :group_id]
+ [:sandboxes.table_id :table_id]
+ [:table.db_id :db_id]
+ [:table.schema :schema]]
+ :from [[:sandboxes]]
+ :left-join [[:metabase_table :table]
+ [:= :sandboxes.table_id :table.id]]
+ :where [:and
+ [:in :sandboxes.group_id group-ids]
+ [:in :table.db_id db-ids]]}))
+
(defn insert-sandbox!
"Insert `sandbox` and return the new instance."
[sandbox]
diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj
index 4ec307fdd4e..1bd34e9c233 100644
--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj
+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj
@@ -1,7 +1,6 @@
(ns metabase-enterprise.sandbox.models.permissions.delete-sandboxes
(:require
[metabase-enterprise.sandbox.db :as sandbox.db]
- [metabase.app-db.core :as app-db]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]))
@@ -45,18 +44,8 @@
:when (contains? db-changes :view-data)]
db-id))]
(when (and (seq all-group-ids) (seq all-db-ids))
- (let [candidate-sandboxes (app-db/query
- {:select [[:sandboxes.id :id]
- [:sandboxes.group_id :group_id]
- [:sandboxes.table_id :table_id]
- [:table.db_id :db_id]
- [:table.schema :schema]]
- :from [[:sandboxes]]
- :left-join [[:metabase_table :table]
- [:= :sandboxes.table_id :table.id]]
- :where [:and
- [:in :sandboxes.group_id all-group-ids]
- [:in :table.db_id all-db-ids]]})
+ (let [candidate-sandboxes (sandbox.db/candidate-sandboxes-for-groups-and-databases
+ all-group-ids all-db-ids)
ids-to-delete (into #{}
(comp (filter (partial should-delete-sandbox? changes))
(map :id))
diff --git a/enterprise/backend/src/metabase_enterprise/security_center/db.clj b/enterprise/backend/src/metabase_enterprise/security_center/db.clj
index 2e65ee16425..49e174667ab 100644
--- a/enterprise/backend/src/metabase_enterprise/security_center/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/security_center/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the security-center module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module only touches `toucan2.core` for model definitions and hydration methods."
(:require
+ [metabase.app-db.core :as mdb]
[toucan2.core :as t2]))
(defn advisory
@@ -14,6 +15,12 @@
[advisory-id]
(t2/select-one :model/SecurityAdvisory :advisory_id advisory-id))
+(defn max-advisory-updated-at
+ "The latest `updated_at` across every SecurityAdvisory, or nil when there are none."
+ []
+ (:updated_at (t2/query-one {:select [[[:max :updated_at] :updated_at]]
+ :from [:security_advisory]})))
+
(defn advisories-newest-first
"Every SecurityAdvisory, newest first."
[]
@@ -44,6 +51,18 @@
[id changes]
(t2/update! :model/SecurityAdvisory id changes))
+(defn upsert-advisory!
+ "Insert or update a SecurityAdvisory by `:advisory_id`. On insert, `:match_status` starts as `:unknown` until the
+ matching engine evaluates it. On update, sets `advisory`'s columns, leaving `:match_status`, `:last_evaluated_at`,
+ and acknowledgement fields (which `advisory` doesn't include) untouched."
+ [advisory]
+ (mdb/update-or-insert! :model/SecurityAdvisory
+ {:advisory_id (:advisory_id advisory)}
+ (fn [existing]
+ (if existing
+ advisory
+ (assoc advisory :match_status :unknown)))))
+
(defn user-summaries-by-id
"A map of ID to the ID, names, and email of the Users with `user-ids`."
[user-ids]
diff --git a/enterprise/backend/src/metabase_enterprise/security_center/fetch.clj b/enterprise/backend/src/metabase_enterprise/security_center/fetch.clj
index f1577e2c73a..d1ce73ba771 100644
--- a/enterprise/backend/src/metabase_enterprise/security_center/fetch.clj
+++ b/enterprise/backend/src/metabase_enterprise/security_center/fetch.clj
@@ -5,8 +5,8 @@
[clojure.edn :as edn]
[clojure.set :as set]
[java-time.api :as t]
+ [metabase-enterprise.security-center.db :as security-center.db]
[metabase-enterprise.security-center.schema :as security-center.schema]
- [metabase.app-db.core :as mdb]
[metabase.config.core :as config]
[metabase.premium-features.core :as premium-features]
[metabase.util.json :as json]
@@ -60,11 +60,7 @@
(defn- latest-updated-at
"Return the maximum `updated_at` across all advisories as an ISO-8601 string, or nil if none exist."
[]
- (some-> (mdb/query {:select [[[:max :updated_at]]]
- :from [:security_advisory]})
- first
- vals
- first
+ (some-> (security-center.db/max-advisory-updated-at)
t/instant
t/format))
@@ -132,12 +128,7 @@
On insert, match_status starts as :unknown until the matching engine evaluates it.
On update, merges new data but preserves :match_status, :last_evaluated_at, and acknowledgement fields."
[advisory]
- (mdb/update-or-insert! :model/SecurityAdvisory
- {:advisory_id (:advisory_id advisory)}
- (fn [existing]
- (if existing
- advisory
- (assoc advisory :match_status :unknown)))))
+ (security-center.db/upsert-advisory! advisory))
(defn sync-advisories!
"Fetch advisories from the MetaStore and upsert into the appdb."
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
index b588dc6c6d9..bfc4e2d6e89 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the transforms-python module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module only touches `toucan2.core` for model definitions and hydration methods."
(:require
+ [metabase.app-db.core :as mdb]
[toucan2.core :as t2]))
(defn table-database-ids
@@ -39,6 +40,13 @@
[path]
(t2/select-one :model/PythonLibrary :path path))
+(defn upsert-python-library-source!
+ "Insert or update the PythonLibrary at `path`, setting its source to `source`. Returns the ID of the row."
+ [path source]
+ (mdb/update-or-insert! :model/PythonLibrary
+ {:path path}
+ (constantly {:path path :source source})))
+
(defn library-sources-by-path
"A map of path to source for every PythonLibrary."
[]
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
index e8529c101bf..712d50ba319 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj
@@ -2,7 +2,6 @@
(:require
[metabase-enterprise.transforms-python.db :as transforms-python.db]
[metabase.api.common :as api]
- [metabase.app-db.core :as app-db]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
@@ -78,9 +77,7 @@
[path source]
(let [normalized-path (normalize-path path)]
(validate-path! normalized-path)
- (let [id (app-db/update-or-insert! :model/PythonLibrary
- {:path normalized-path}
- (constantly {:path normalized-path :source source}))]
+ (let [id (transforms-python.db/upsert-python-library-source! normalized-path source)]
(transforms-python.db/python-library id))))
;;; ------------------------------------------------- Serialization --------------------------------------------------
diff --git a/src/metabase/app_db/db.clj b/src/metabase/app_db/db.clj
index a30a97dc329..27a6966cd4d 100644
--- a/src/metabase/app_db/db.clj
+++ b/src/metabase/app_db/db.clj
@@ -6,8 +6,37 @@
first, and while rows are being re-encrypted a setting row can be plaintext under a key, or ciphertext under a key
not yet in effect, which that model's strict read rejects."
(:require
+ [honey.sql :as sql]
+ [metabase.util.honey-sql-2 :as h2x]
[toucan2.core :as t2]))
+(defn current-timestamp-string
+ "The application DB's own current timestamp, as a string, for app DB type `db-type`."
+ ^String [db-type]
+ ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)
+ (let [cast-form (h2x/cast (if (= db-type :mysql) :char :text) (h2x/current-datetime-honeysql-form db-type))]
+ (:timestamp (t2/query-one {:select [[cast-form :timestamp]]}))))
+
+;;; ------------------------------------------------ Liquibase ------------------------------------------------
+
+(defn changelog-by-id
+ "The Liquibase changelog row with `changelog-id` in the app DB of type `db-type`, or nil."
+ [db-type changelog-id]
+ (let [table-name (case db-type
+ (:postgres :h2) "databasechangelog"
+ :mysql "DATABASECHANGELOG")]
+ (t2/query-one [(format "select * from %s where id = ?" table-name) changelog-id])))
+
+(defn changelog-ids
+ "The ids among `changelog-ids` still present in the Liquibase changelog table `changelog-table-name`, read on
+ `conn`."
+ [conn changelog-table-name changelog-ids]
+ (map :id (t2/query conn (sql/format {:select [:id]
+ :from [(keyword changelog-table-name)]
+ :where [:in :id changelog-ids]}))))
+
+;;; ------------------------------------------------ Settings ------------------------------------------------
+
(def ^:private unmigrated-settings-where
[:and [:= :value_with_aad nil] [:not= :value nil] [:not= :value ""]])
diff --git a/src/metabase/app_db/liquibase.clj b/src/metabase/app_db/liquibase.clj
index 4afbb4f4467..c81e9675953 100644
--- a/src/metabase/app_db/liquibase.clj
+++ b/src/metabase/app_db/liquibase.clj
@@ -3,10 +3,9 @@
(:require
[clojure.java.jdbc :as jdbc]
[clojure.string :as str]
- [honey.sql :as sql]
- [honey.sql.helpers :as sql.helpers]
[metabase.app-db.connection :as mdb.connection]
[metabase.app-db.custom-migrations]
+ [metabase.app-db.db :as app-db.db]
[metabase.app-db.liquibase.h2 :as liquibase.h2]
[metabase.app-db.liquibase.mysql :as liquibase.mysql]
[metabase.classloader.core :as classloader]
@@ -16,8 +15,7 @@
[metabase.util.log :as log]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
- [toucan2.connection :as t2.conn]
- [toucan2.core :as t2])
+ [toucan2.connection :as t2.conn])
(:import
(java.io StringWriter)
(java.sql Connection)
@@ -176,10 +174,7 @@
(defn changelog-by-id
"Return the changelog row value for the given `changelog-id`."
[app-db changelog-id]
- (let [table-name (case (:db-type app-db)
- (:postgres :h2) "databasechangelog"
- :mysql "DATABASECHANGELOG")]
- (t2/query-one (format "select * from %s where id = '%s'" table-name changelog-id))))
+ (app-db.db/changelog-by-id (:db-type app-db) changelog-id))
(defn migrations-sql
"Return a string of SQL containing the DDL statements needed to perform unrun `liquibase` migrations, custom migrations will be ignored."
@@ -597,11 +592,7 @@
(.getChangeLogParameters liquibase)
changelog
change-listener))
- (let [remaining-query (-> (sql.helpers/select :id)
- (sql.helpers/from (keyword (changelog-table-name liquibase)))
- (sql.helpers/where [:in :id ids-to-drop]))
- formatted-sql (sql/format remaining-query)
- remaining-ids (map :id (t2/query conn formatted-sql))]
+ (let [remaining-ids (app-db.db/changelog-ids conn (changelog-table-name liquibase) ids-to-drop)]
(when (seq remaining-ids)
(log/warnf "The following changesets were not rolled back. Likely because %s: %s"
(if (seq @error-ids)
diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.clj
index 5580cdef999..c389528700d 100644
--- a/src/metabase/app_db/query.clj
+++ b/src/metabase/app_db/query.clj
@@ -22,9 +22,9 @@
(:require
[clojure.string :as str]
[honey.sql :as sql]
+ [metabase.app-db.db :as app-db.db]
[metabase.app-db.format :as app-db.format]
[metabase.util :as u]
- [metabase.util.honey-sql-2 :as h2x]
[metabase.util.malli :as mu]
[metabase.util.malli.schema :as ms]
[toucan2.core :as t2]
@@ -121,9 +121,7 @@
"The application DB's own current timestamp, as a string. Read from the DB rather than from this machine's clock,
for the `settings-last-updated` marker that instances compare against each other."
^String [db-type]
- ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)
- (let [cast-form (h2x/cast (if (= db-type :mysql) :char :text) (h2x/current-datetime-honeysql-form db-type))]
- (-> (t2/query-one {:select [[cast-form :timestamp]]}) :timestamp)))
+ (app-db.db/current-timestamp-string db-type))
;;; TODO -- we should mark this deprecated and tell people to use [[toucan2.core/query]] directly instead
(defn query
diff --git a/src/metabase/cache/db.clj b/src/metabase/cache/db.clj
index 6a71ee4bf1e..310c211d47d 100644
--- a/src/metabase/cache/db.clj
+++ b/src/metabase/cache/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the cache module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
+ [metabase.app-db.core :as app-db]
[toucan2.core :as t2]))
(defn database-with-ids
@@ -103,6 +104,12 @@
[model model-id]
(t2/select-one :model/CacheConfig :model model :model_id model-id {:for :update}))
+(defn upsert-cache-config!
+ "Insert or replace the CacheConfig for `model` and `model-id` with `data`."
+ [model model-id data]
+ (app-db/update-or-insert! :model/CacheConfig {:model model :model_id model-id}
+ (constantly data)))
+
(defn cache-configs-for
"The CacheConfigs for `model` and `model-ids`."
[model model-ids]
diff --git a/src/metabase/cache/models/cache_config.clj b/src/metabase/cache/models/cache_config.clj
index 30d08e1dcab..4e2c419ecf9 100644
--- a/src/metabase/cache/models/cache_config.clj
+++ b/src/metabase/cache/models/cache_config.clj
@@ -3,7 +3,6 @@
(:require
[java-time.api :as t]
[medley.core :as m]
- [metabase.app-db.core :as app-db]
[metabase.cache.db :as cache.db]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
@@ -168,8 +167,7 @@
(t2/with-transaction [_tx]
(let [data (config->row config)
current (cache.db/lock-cache-config model model_id)]
- (u/prog1 (app-db/update-or-insert! :model/CacheConfig {:model model :model_id model_id}
- (constantly data))
+ (u/prog1 (cache.db/upsert-cache-config! model model_id data)
(audit-caching-change! user-id <> current data)))))
(defn delete!
diff --git a/src/metabase/channel/db.clj b/src/metabase/channel/db.clj
index 59af45f697a..6ab6a68e83b 100644
--- a/src/metabase/channel/db.clj
+++ b/src/metabase/channel/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the channel module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
+ [metabase.app-db.core :as app-db]
[toucan2.core :as t2]))
(defn channels
@@ -56,6 +57,20 @@
[:= :is_active true]
[:in :id user-ids]]}))
+(defn user-ids-with-permission
+ "The ids of the Users belonging to a PermissionsGroup that holds `permission-path`."
+ [permission-path]
+ (app-db/query {:select [:pgm.user_id]
+ :from [[:permissions_group_membership :pgm]]
+ :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]]
+ :where [:exists ^:allow-subquery
+ {:select [1]
+ :from [[:permissions :p]]
+ :where [:and
+ [:= :p.group_id :pg.id]
+ [:= :p.object permission-path]]}]
+ :group-by [:pgm.user_id]}))
+
(defn delete-pulse-channels-for-channel!
"Delete the PulseChannels of the Channel with `channel-id`."
[channel-id]
diff --git a/src/metabase/channel/email/messages.clj b/src/metabase/channel/email/messages.clj
index 7e0500758a3..6fe6243d4c3 100644
--- a/src/metabase/channel/email/messages.clj
+++ b/src/metabase/channel/email/messages.clj
@@ -8,7 +8,6 @@
[clojure.string :as str]
[java-time.api :as t]
[medley.core :as m]
- [metabase.app-db.core :as app-db]
[metabase.appearance.core :as appearance]
[metabase.channel.db :as channel.db]
[metabase.channel.email :as email]
@@ -249,19 +248,7 @@
[database-id]
(let [monitoring (perms/application-perms-path :monitoring)
user-ids-with-monitoring (when (premium-features/enable-advanced-permissions?)
- (->> {:select [:pgm.user_id]
- :from [[:permissions_group_membership :pgm]]
- :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]]
- :where [:and
- [:exists ^:allow-subquery
- {:select [1]
- :from [[:permissions :p]]
- :where [:and
- [:= :p.group_id :pg.id]
- [:= :p.object monitoring]]}]]
- :group-by [:pgm.user_id]}
- app-db/query
- (mapv :user_id)))
+ (mapv :user_id (channel.db/user-ids-with-permission monitoring)))
user-ids (filter
#(perms/user-has-permission-for-database? % :perms/manage-database :yes database-id)
user-ids-with-monitoring)]
diff --git a/src/metabase/collections_rest/api.clj b/src/metabase/collections_rest/api.clj
index 776e57298de..ca53d356797 100644
--- a/src/metabase/collections_rest/api.clj
+++ b/src/metabase/collections_rest/api.clj
@@ -1114,7 +1114,7 @@
[rows rows-query offset]
(or (some-> rows first :total_count)
(when (pos? (or offset 0))
- (some-> (mdb/query (assoc rows-query :limit 1)) first :total_count))
+ (some-> (collections-rest.db/collection-children-rows (assoc rows-query :limit 1)) first :total_count))
0))
(defn- collection-children*
@@ -1157,7 +1157,7 @@
:limit (if (zero? limit) 1 limit)
:offset offset))
rows (tracing/with-span :db-app "db-app.collection-items-query" {:collection/id (:id collection)}
- (mdb/query limit-query))
+ (collections-rest.db/collection-children-rows limit-query))
res {:total (total-count rows rows-query offset)
:data (if (= limit 0)
[]
@@ -1219,7 +1219,7 @@
:permission-level (if archived? :write :read)
:include-trash-collection? archived?}
row (first
- (mdb/query
+ (collections-rest.db/collection-filter-metadata-rows
{:with [[:visible_collection_ids (collection/visible-collection-query viz-config)]]
:select (vec
(for [model candidates]
diff --git a/src/metabase/collections_rest/db.clj b/src/metabase/collections_rest/db.clj
index 08112c56896..a5a294c4345 100644
--- a/src/metabase/collections_rest/db.clj
+++ b/src/metabase/collections_rest/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the collections REST module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module only touches `toucan2.core` for hydration."
(:require
+ [metabase.app-db.core :as app-db]
[metabase.collections.models.collection :as collection]
[toucan2.core :as t2]))
@@ -165,3 +166,17 @@
"The Cards in the Collection with `collection-id`."
[collection-id]
(t2/select :model/Card :collection_id collection-id))
+
+(defn collection-children-rows
+ "The rows matching the collection-children Honey SQL `query`, built by `metabase.collections-rest.api` from the
+ per-model item queries for a Collection's paginated child listing. Follows the same exception as
+ `metabase.search.db` for spec-driven Honey SQL that can't be reduced to plain-data parameters."
+ [query]
+ (app-db/query query))
+
+(defn collection-filter-metadata-rows
+ "The rows matching the collection-filter-metadata Honey SQL `query`, built by `metabase.collections-rest.api` to
+ probe which item models have at least one visible child in a Collection. Follows the same exception as
+ `metabase.search.db` for spec-driven Honey SQL that can't be reduced to plain-data parameters."
+ [query]
+ (app-db/query query))
diff --git a/src/metabase/content_verification/db.clj b/src/metabase/content_verification/db.clj
index ea3c42a9c36..e7048ba4c4a 100644
--- a/src/metabase/content_verification/db.clj
+++ b/src/metabase/content_verification/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the content verification module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
+ [metabase.app-db.core :as app-db]
[toucan2.core :as t2]))
(defn moderation-reviews-for-items
@@ -17,6 +18,16 @@
[user-ids]
(t2/select :model/User :id [:in user-ids]))
+(defn moderation-review-ids-for-item
+ "The ids of the ModerationReviews of the item with `item-id` and `item-type`, newest first."
+ [item-id item-type]
+ (app-db/query {:select [:id]
+ :from [:moderation_review]
+ :where [:and
+ [:= :moderated_item_id item-id]
+ [:= :moderated_item_type item-type]]
+ :order-by [[:id :desc]]}))
+
(defn most-recent-moderation-review-statuses
"The item id, item type, and status of the most recent ModerationReviews of the items with `item-types` and
`item-ids`, newest first."
diff --git a/src/metabase/content_verification/models/moderation_review.clj b/src/metabase/content_verification/models/moderation_review.clj
index 9245dc32633..b9b023b90dc 100644
--- a/src/metabase/content_verification/models/moderation_review.clj
+++ b/src/metabase/content_verification/models/moderation_review.clj
@@ -1,6 +1,5 @@
(ns metabase.content-verification.models.moderation-review
(:require
- [metabase.app-db.core :as app-db]
[metabase.content-verification.db :as content-verification.db]
[metabase.content-verification.impl :as moderation]
[metabase.models.interface :as mi]
@@ -49,18 +48,12 @@
ensures there are one fewer than that so you can add afterwards."
[item-id :- :int
item-type :- :string]
- (let [ids (into #{} (comp (map :id)
+ (let [;; cannot put the offset in this query as mysql doesn't play nice. It requires a limit as well which we do
+ ;; not want to give. The offset is only 10 though so its not a huge savings and we run this on every entry so
+ ;; the max number is 10, delete the extra, and insert a new one to arrive at 10 again, our invariant.
+ ids (into #{} (comp (map :id)
(drop (dec max-moderation-reviews)))
- (app-db/query {:select [:id]
- :from [:moderation_review]
- :where [:and
- [:= :moderated_item_id item-id]
- [:= :moderated_item_type item-type]]
- ;; cannot put the offset in this query as mysql doesn't play nice. It requires a limit
- ;; as well which we do not want to give. The offset is only 10 though so its not a huge
- ;; savings and we run this on every entry so the max number is 10, delete the extra,
- ;; and insert a new one to arrive at 10 again, our invariant.
- :order-by [[:id :desc]]}))]
+ (content-verification.db/moderation-review-ids-for-item item-id item-type))]
(when (seq ids)
(content-verification.db/delete-moderation-reviews! ids))))
diff --git a/src/metabase/dashboards/db.clj b/src/metabase/dashboards/db.clj
index 0c0f47843a2..7b0c24afcaf 100644
--- a/src/metabase/dashboards/db.clj
+++ b/src/metabase/dashboards/db.clj
@@ -40,6 +40,14 @@
:dashboard_id [:in dashboard-ids]
{:order-by [[:dashboard_id :asc] [:position :asc] [:id :asc]]}))
+(defn dashboard-collection-authority-levels
+ "The id and Collection `authority_level` of the Dashboards with `dashboard-ids`."
+ [dashboard-ids]
+ (mdb/query {:select [:dashboard.id :collection.authority_level]
+ :from [[:report_dashboard :dashboard]]
+ :left-join [[:collection :collection] [:= :collection.id :dashboard.collection_id]]
+ :where [:in :dashboard.id dashboard-ids]}))
+
(defn dashcards-with-visible-cards-for-dashboards
"The DashboardCards of the Dashboards with `dashboard-ids` whose Card is visible (unarchived, a dashboard question
not archived by itself, or absent), with their Card's Collection authority level, in dashboard then creation order."
@@ -138,6 +146,19 @@
[dashcard-id]
(t2/select-one :model/DashboardCard :id dashcard-id))
+(defn multi-cards-for-dashcard
+ "The unarchived Cards added to the DashboardCard with `dashcard-id` as series (the 'add series' feature)."
+ [dashcard-id]
+ (mdb/query {:select [:newcard.*]
+ :from [[:report_dashboardcard :dashcard]]
+ :left-join [[:dashboardcard_series :dashcardseries]
+ [:= :dashcard.id :dashcardseries.dashboardcard_id]
+ [:report_card :newcard]
+ [:= :dashcardseries.card_id :newcard.id]]
+ :where [:and
+ [:= :newcard.archived false]
+ [:= :dashcard.id dashcard-id]]}))
+
(defn dashcards-by-ids
"The DashboardCards with `dashcard-ids`."
[dashcard-ids]
diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.clj
index bda35299c77..96d10850ae6 100644
--- a/src/metabase/dashboards/models/dashboard.clj
+++ b/src/metabase/dashboards/models/dashboard.clj
@@ -3,7 +3,6 @@
[clojure.set :as set]
[medley.core :as m]
[metabase.api.common :as api]
- [metabase.app-db.core :as app-db]
[metabase.audit-app.core :as audit]
[metabase.collections.core :as collections]
[metabase.collections.models.collection :as collection]
@@ -164,10 +163,8 @@
(when (seq dashboards)
(let [coll-id->level (into {}
(map (juxt :id :authority_level))
- (app-db/query {:select [:dashboard.id :collection.authority_level]
- :from [[:report_dashboard :dashboard]]
- :left-join [[:collection :collection] [:= :collection.id :dashboard.collection_id]]
- :where [:in :dashboard.id (into #{} (map u/the-id) dashboards)]}))]
+ (dashboards.db/dashboard-collection-authority-levels
+ (into #{} (map u/the-id) dashboards)))]
(for [dashboard dashboards]
(assoc dashboard :collection_authority_level (get coll-id->level (u/the-id dashboard)))))))
diff --git a/src/metabase/dashboards/models/dashboard_card.clj b/src/metabase/dashboards/models/dashboard_card.clj
index 7cbf17bc87a..26fb504af46 100644
--- a/src/metabase/dashboards/models/dashboard_card.clj
+++ b/src/metabase/dashboards/models/dashboard_card.clj
@@ -2,7 +2,6 @@
(:require
[clojure.set :as set]
[medley.core :as m]
- [metabase.app-db.core :as mdb]
[metabase.dashboards.db :as dashboards.db]
[metabase.models.interface :as mi]
[metabase.models.serialization :as serdes]
@@ -159,15 +158,7 @@
This is also different from having multiple series displayed on Line, Area, or Bar Questions."
[dashcard]
- (mdb/query {:select [:newcard.*]
- :from [[:report_dashboardcard :dashcard]]
- :left-join [[:dashboardcard_series :dashcardseries]
- [:= :dashcard.id :dashcardseries.dashboardcard_id]
- [:report_card :newcard]
- [:= :dashcardseries.card_id :newcard.id]]
- :where [:and
- [:= :newcard.archived false]
- [:= :dashcard.id (:id dashcard)]]}))
+ (dashboards.db/multi-cards-for-dashcard (:id dashcard)))
(defn update-dashboard-cards-series!
"Batch update the DashboardCardSeries for multiple DashboardCards.
diff --git a/src/metabase/dashboards_rest/api.clj b/src/metabase/dashboards_rest/api.clj
index ba3d288e5f9..a084f2ca5a6 100644
--- a/src/metabase/dashboards_rest/api.clj
+++ b/src/metabase/dashboards_rest/api.clj
@@ -10,7 +10,6 @@
[metabase.analytics.core :as analytics]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
- [metabase.app-db.core :as app-db]
[metabase.channel.email.messages :as messages]
[metabase.channel.render.core :as channel.render]
[metabase.collections-rest.api :as api.collection]
@@ -49,7 +48,6 @@
[metabase.request.core :as request]
[metabase.revisions.core :as revisions]
[metabase.util :as u]
- [metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [deferred-tru tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
@@ -715,34 +713,7 @@
;; Output should match the shape of api/collection/<:id|root>/items. There's a test that asserts that this remains
;; the case, but if you change one, you'll want to change both.
(let [dashboard (api/read-check :model/Dashboard id)
- query (merge
- {:select [:c.id :c.name :c.description :c.entity_id :c.collection_position :c.display :c.collection_preview
- :last_used_at :c.collection_id :c.archived_directly :c.archived :c.database_id
- :c.dashboard_id
- [nil :location]
- [(h2x/literal "card") :model]
- [^:allow-subquery {:select [:status]
- :from [:moderation_review]
- :where [:and
- [:= :moderated_item_type "card"]
- [:= :moderated_item_id :c.id]
- [:= :most_recent true]]
- ;; limit 1 to ensure that there is only one result but this invariant should hold true, just
- ;; protecting against potential bugs
- :order-by [[:id :desc]]
- :limit 1}
- :moderated_status]]
- :from [[:report_card :c]]
- :where [:and
- [:= :c.dashboard_id id]
- [:exists ^:allow-subquery {:select 1
- :from [[:report_dashboardcard :dc]]
- :where [:and [:= :c.id :dc.card_id] [:= :c.dashboard_id :dc.dashboard_id]]}]
- [:= :c.archived false]]}
- (when (request/paged?)
- {:limit (request/limit)
- :offset (request/offset)}))
- cards (app-db/query query)]
+ cards (dashboards-rest.db/dashboard-item-cards id (request/paged?) (request/limit) (request/offset))]
{:total (count cards)
:data (api.collection/post-process-rows {}
(dashboards-rest.db/collection (:collection_id dashboard))
diff --git a/src/metabase/dashboards_rest/db.clj b/src/metabase/dashboards_rest/db.clj
index 924e24dcb6c..2bc7b035766 100644
--- a/src/metabase/dashboards_rest/db.clj
+++ b/src/metabase/dashboards_rest/db.clj
@@ -2,6 +2,8 @@
"Application database queries for the dashboards REST module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module only touches `toucan2.core` for hydration."
(:require
+ [metabase.app-db.core :as app-db]
+ [metabase.util.honey-sql-2 :as h2x]
[toucan2.core :as t2]))
(defn dashboards
@@ -70,6 +72,37 @@
[dashboard-id]
(t2/select-fn-vec :card_id :model/DashboardCard :dashboard_id dashboard-id))
+(defn dashboard-item-cards
+ "The card items of the Dashboard with `dashboard-id`, for the `/:id/items` endpoint: id, name, description, entity
+ id, collection position, display, collection preview, last-used-at, collection id, archived flags, database id,
+ and moderated status. Paged by `limit`/`offset` when `paged?`."
+ [dashboard-id paged? limit offset]
+ (app-db/query
+ (cond-> {:select [:c.id :c.name :c.description :c.entity_id :c.collection_position :c.display :c.collection_preview
+ :last_used_at :c.collection_id :c.archived_directly :c.archived :c.database_id
+ :c.dashboard_id
+ [nil :location]
+ [(h2x/literal "card") :model]
+ [^:allow-subquery {:select [:status]
+ :from [:moderation_review]
+ :where [:and
+ [:= :moderated_item_type "card"]
+ [:= :moderated_item_id :c.id]
+ [:= :most_recent true]]
+ ;; limit 1 to ensure that there is only one result but this invariant should
+ ;; hold true, just protecting against potential bugs
+ :order-by [[:id :desc]]
+ :limit 1}
+ :moderated_status]]
+ :from [[:report_card :c]]
+ :where [:and
+ [:= :c.dashboard_id dashboard-id]
+ [:exists ^:allow-subquery {:select 1
+ :from [[:report_dashboardcard :dc]]
+ :where [:and [:= :c.id :dc.card_id] [:= :c.dashboard_id :dc.dashboard_id]]}]
+ [:= :c.archived false]]}
+ paged? (merge {:limit limit :offset offset}))))
+
(defn dashboard-series-card-ids
"The Card ids of the DashboardCardSeries of the Dashboard with `dashboard-id`."
[dashboard-id]
diff --git a/src/metabase/mcp/db.clj b/src/metabase/mcp/db.clj
index 16227ccf50e..0a56168750b 100644
--- a/src/metabase/mcp/db.clj
+++ b/src/metabase/mcp/db.clj
@@ -2,6 +2,8 @@
"Application database queries for the MCP module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
+ [metabase.app-db.core :as app-db]
+ [metabase.session.core :as session]
[toucan2.core :as t2]))
(defn insert-feedback!
@@ -14,6 +16,20 @@
[key-hashed]
(t2/select-one-fn :user_id :core_session :key_hashed key-hashed))
+(defn get-or-create-core-session!
+ "The `core_session` row for `key-hashed` and `user-id`, creating one with a freshly generated session id if none
+ exists. Uses the raw `:core_session` table (not `:model/Session`) to bypass the after-insert hook, which would
+ otherwise publish a spurious `:event/user-login` event."
+ [key-hashed user-id]
+ (app-db/select-or-insert!
+ :core_session
+ {:key_hashed key-hashed
+ :user_id user-id}
+ (fn []
+ {:id (session/generate-session-id)
+ :anti_csrf_token nil
+ :created_at :%now})))
+
(defn insert-query-handle!
"Insert the McpQueryHandle `row`."
[row]
diff --git a/src/metabase/mcp/session.clj b/src/metabase/mcp/session.clj
index c170d2e4416..facbef54544 100644
--- a/src/metabase/mcp/session.clj
+++ b/src/metabase/mcp/session.clj
@@ -13,7 +13,6 @@
TTL and is re-created on demand for query-handle lifecycle management."
(:require
[clojure.string :as str]
- [metabase.app-db.core :as app-db]
[metabase.mcp.db :as mcp.db]
[metabase.mcp.models.mcp-query-handle]
[metabase.mcp.settings :as mcp.settings]
@@ -291,17 +290,7 @@
;; colliding rows arbitrarily. Cross-user UUID collisions are an unaddressed risk
;; across the whole session model (cookie sessions included) — it belongs in the
;; auth layer, not here, and no constraint shape at this call site can fix it.
- ;;
- ;; Raw :core_session (not :model/Session) to bypass the after-insert hook, which
- ;; would publish spurious :event/user-login events.
- (app-db/select-or-insert!
- :core_session
- {:key_hashed key-hashed
- :user_id user-id}
- (fn []
- {:id (session/generate-session-id)
- :anti_csrf_token nil
- :created_at :%now}))))
+ (mcp.db/get-or-create-core-session! key-hashed user-id)))
(defn get-or-create-session-key!
"Ensure a `core_session` exists for this MCP session and return its (plaintext)
diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.clj
index 1c1d0d4c040..cef75ebc65c 100644
--- a/src/metabase/metabot/db.clj
+++ b/src/metabase/metabot/db.clj
@@ -234,6 +234,12 @@
[conversation]
(t2/insert! :model/MetabotConversation conversation))
+(defn upsert-conversation!
+ "Insert or update the MetabotConversation with `conversation-id`. `update-fn` receives the existing row (or nil on
+ insert) and must return the fields to write."
+ [conversation-id update-fn]
+ (mdb/update-or-insert! :model/MetabotConversation {:id conversation-id} update-fn))
+
(defn set-conversation-title-if-missing!
"Set the title of the MetabotConversation with `conversation-id` if it has none."
[conversation-id title]
@@ -310,6 +316,29 @@
[rows]
(t2/insert! :model/MetabotUsedTable rows))
+;;; --------------------------------------------------- Feedback ---------------------------------------------------
+
+(defn upsert-feedback!
+ "Insert or update the MetabotFeedback row for the MetabotMessage with `message-id` and the User with
+ `submitter-user-id`. `update-fn` receives the existing row (or nil on insert) and must return the fields to
+ write."
+ [message-id submitter-user-id update-fn]
+ (mdb/update-or-insert! :model/MetabotFeedback
+ {:message_id message-id :user_id submitter-user-id}
+ update-fn))
+
+(defn upsert-source-feedback!
+ "Insert or update the MetabotSourceFeedback row for the MetabotMessage with `message-id`, the User with
+ `submitter-user-id`, and the source with `source-id`/`source-type`. `update-fn` receives the existing row (or nil
+ on insert) and must return the fields to write."
+ [message-id submitter-user-id source-id source-type update-fn]
+ (mdb/update-or-insert! :model/MetabotSourceFeedback
+ {:message_id message-id
+ :user_id submitter-user-id
+ :source_id source-id
+ :source_type source-type}
+ update-fn))
+
;;; ------------------------------------------------ Databases ------------------------------------------------
(defn database-summary
diff --git a/src/metabase/metabot/feedback.clj b/src/metabase/metabot/feedback.clj
index babd98c5c40..b85bcf43756 100644
--- a/src/metabase/metabot/feedback.clj
+++ b/src/metabase/metabot/feedback.clj
@@ -2,7 +2,6 @@
"Local persistence of Metabot feedback."
(:require
[metabase.api.common :as api]
- [metabase.app-db.core :as app-db]
[metabase.metabot.db :as metabot.db]
[metabase.models.interface :as mi]))
@@ -27,25 +26,19 @@
(let [base-fields {:positive positive
:issue_type issue_type
:freeform_feedback freeform_feedback}]
- (app-db/update-or-insert! :model/MetabotFeedback
- {:message_id message-row-id :user_id submitter-user-id}
- (fn [existing]
- (cond-> base-fields
- existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))
+ (metabot.db/upsert-feedback! message-row-id submitter-user-id
+ (fn [existing]
+ (cond-> base-fields
+ existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))
(defn- upsert-source-feedback!
"Insert or update the `metabot_source_feedback` row for one source, message, and submitter."
[message-row-id submitter-user-id {:keys [positive source_id source_type]}]
- (let [conditions {:message_id message-row-id
- :user_id submitter-user-id
- :source_id source_id
- :source_type source_type}
- base-fields {:positive positive}]
- (app-db/update-or-insert! :model/MetabotSourceFeedback
- conditions
- (fn [existing]
- (cond-> base-fields
- existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))
+ (let [base-fields {:positive positive}]
+ (metabot.db/upsert-source-feedback! message-row-id submitter-user-id source_id source_type
+ (fn [existing]
+ (cond-> base-fields
+ existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))
(defn persist-feedback!
"Upsert a `metabot_feedback` row for the rated message and return the
diff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.clj
index c3ea40eff11..dd0110dc441 100644
--- a/src/metabase/metabot/persistence.clj
+++ b/src/metabase/metabot/persistence.clj
@@ -4,7 +4,6 @@
[clojure.string :as str]
[metabase.analytics-interface.core :as analytics]
[metabase.api.common :as api]
- [metabase.app-db.core :as app-db]
[metabase.llm.provider :as llm.provider]
[metabase.metabot.agent.memory :as memory]
[metabase.metabot.agent.streaming :as streaming]
@@ -276,28 +275,29 @@
(t2/with-transaction [_conn]
(when (seq delete-message-ids)
(soft-delete-messages! {:id [:in delete-message-ids]} originator-id))
- (app-db/update-or-insert! :model/MetabotConversation {:id conversation-id}
- (fn [existing]
- ;; `:user_id` is the originator — set on insert, never overwritten.
- (cond-> {}
- (nil? existing)
- (assoc :user_id originator-id)
- (and hostname (nil? (:embedding_hostname existing)))
- (assoc :embedding_hostname hostname)
- (and (:embedding_path pii-info) (nil? (:embedding_path existing)))
- (assoc :embedding_path (:embedding_path pii-info))
- (and (:user_agent pii-info) (nil? (:user_agent existing)))
- (assoc :user_agent (:user_agent pii-info))
- (and (:sanitized_user_agent pii-info) (nil? (:sanitized_user_agent existing)))
- (assoc :sanitized_user_agent (:sanitized_user_agent pii-info))
- (and (:ip_address pii-info) (nil? (:ip_address existing)))
- (assoc :ip_address (:ip_address pii-info))
- (and slack-team-id (nil? (:slack_team_id existing)))
- (assoc :slack_team_id slack-team-id)
- (and channel-id (nil? (:slack_channel_id existing)))
- (assoc :slack_channel_id channel-id)
- (and slack-thread-ts (nil? (:slack_thread_ts existing)))
- (assoc :slack_thread_ts slack-thread-ts))))
+ (metabot.db/upsert-conversation!
+ conversation-id
+ (fn [existing]
+ ;; `:user_id` is the originator — set on insert, never overwritten.
+ (cond-> {}
+ (nil? existing)
+ (assoc :user_id originator-id)
+ (and hostname (nil? (:embedding_hostname existing)))
+ (assoc :embedding_hostname hostname)
+ (and (:embedding_path pii-info) (nil? (:embedding_path existing)))
+ (assoc :embedding_path (:embedding_path pii-info))
+ (and (:user_agent pii-info) (nil? (:user_agent existing)))
+ (assoc :user_agent (:user_agent pii-info))
+ (and (:sanitized_user_agent pii-info) (nil? (:sanitized_user_agent existing)))
+ (assoc :sanitized_user_agent (:sanitized_user_agent pii-info))
+ (and (:ip_address pii-info) (nil? (:ip_address existing)))
+ (assoc :ip_address (:ip_address pii-info))
+ (and slack-team-id (nil? (:slack_team_id existing)))
+ (assoc :slack_team_id slack-team-id)
+ (and channel-id (nil? (:slack_channel_id existing)))
+ (assoc :slack_channel_id channel-id)
+ (and slack-thread-ts (nil? (:slack_thread_ts existing)))
+ (assoc :slack_thread_ts slack-thread-ts))))
(metabot.db/insert-messages!
(cond-> {:conversation_id conversation-id
:data (schema.v2/check-message-data "metabot_message.data"
diff --git a/src/metabase/osi/ai_context/api.clj b/src/metabase/osi/ai_context/api.clj
index 500ed939fc9..e700e71ab48 100644
--- a/src/metabase/osi/ai_context/api.clj
+++ b/src/metabase/osi/ai_context/api.clj
@@ -4,7 +4,6 @@
(:require
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
- [metabase.app-db.core :as app-db]
[metabase.entity-retrieval.core :as entity-retrieval]
[metabase.osi.db :as osi.db]
[metabase.osi.models.osi-ai-context :as osi-ai-context]
@@ -110,10 +109,7 @@
;; Upsert on the normalized (stored) key so re-posting a relabelled card updates its one row.
;; update-or-insert! handles the compound key, the no-op re-PUT, and the concurrent-create race
;; (savepoint + single retry) centrally.
- (app-db/update-or-insert! :model/OsiAiContext
- {:entity_type (entity-retrieval/normalize-entity-type entity-type)
- :entity_local_id entity-local-id}
- (constantly {:ai_context ai_context}))
+ (osi.db/upsert-ai-context! (entity-retrieval/normalize-entity-type entity-type) entity-local-id ai_context)
(get-entry entity-type entity-local-id))
(api.macros/defendpoint :post "/reconcile"
diff --git a/src/metabase/osi/db.clj b/src/metabase/osi/db.clj
index 0c6a4a45244..7671c8ae42a 100644
--- a/src/metabase/osi/db.clj
+++ b/src/metabase/osi/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the OSI module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
+ [metabase.app-db.core :as app-db]
[toucan2.core :as t2]))
(defn ai-context
@@ -31,3 +32,12 @@
"Apply `changes` to the OsiAiContext of the entity with `entity-type` and `entity-local-id`."
[entity-type entity-local-id changes]
(t2/update! :model/OsiAiContext :entity_type entity-type :entity_local_id entity-local-id changes))
+
+(defn upsert-ai-context!
+ "Insert or replace the `:ai_context` of the OsiAiContext of the entity with `entity-type` and `entity-local-id`
+ with `ai-context`."
+ [entity-type entity-local-id ai-context]
+ (app-db/update-or-insert! :model/OsiAiContext
+ {:entity_type entity-type
+ :entity_local_id entity-local-id}
+ (constantly {:ai_context ai-context})))
diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.clj
index 849eb2b03c9..dbd98dbbb42 100644
--- a/src/metabase/parameters/chain_filter.clj
+++ b/src/metabase/parameters/chain_filter.clj
@@ -201,22 +201,7 @@
(u/minutes->ms 5))
(defn- database-fk-relationships* [database-id enable-reverse-joins?]
- (let [rows (mdb/query {:select [[:fk-field.id :f1]
- [:fk-table.id :t1]
- [:pk-field.id :f2]
- [:pk-field.table_id :t2]]
- :from [[:metabase_field :fk-field]]
- :left-join [[:metabase_table :fk-table] [:and [:= :fk-field.table_id :fk-table.id]
- :fk-table.active]
- [:metabase_database :database] [:= :fk-table.db_id :database.id]
- [:metabase_field :pk-field] [:and [:= :fk-field.fk_target_field_id :pk-field.id]
- :pk-field.active]]
- :where [:and
- [:= :database.id database-id]
- [:not= :fk-field.fk_target_field_id nil]
- :fk-field.active]
- :order-by [[:fk-field.id :desc]
- [:pk-field.id :desc]]})
+ (let [rows (parameters.db/fk-relationships-for-database database-id)
joins (for [{:keys [t1 f1 t2 f2]} rows]
{:lhs {:table t1, :field f1}
:rhs {:table t2, :field f2}})
diff --git a/src/metabase/parameters/db.clj b/src/metabase/parameters/db.clj
index 9f84f968a92..ddffb7538f6 100644
--- a/src/metabase/parameters/db.clj
+++ b/src/metabase/parameters/db.clj
@@ -138,3 +138,24 @@
"The `columns` of the Fields with `field-ids`."
[columns field-ids]
(t2/select (into [:model/Field] columns) :id [:in field-ids]))
+
+(defn fk-relationships-for-database
+ "Rows describing FK -> PK Field relationships (`:f1`/`:t1` FK Field/Table ids, `:f2`/`:t2` PK Field/Table ids)
+ for active Fields in the Database with `database-id`."
+ [database-id]
+ (mdb/query {:select [[:fk-field.id :f1]
+ [:fk-table.id :t1]
+ [:pk-field.id :f2]
+ [:pk-field.table_id :t2]]
+ :from [[:metabase_field :fk-field]]
+ :left-join [[:metabase_table :fk-table] [:and [:= :fk-field.table_id :fk-table.id]
+ :fk-table.active]
+ [:metabase_database :database] [:= :fk-table.db_id :database.id]
+ [:metabase_field :pk-field] [:and [:= :fk-field.fk_target_field_id :pk-field.id]
+ :pk-field.active]]
+ :where [:and
+ [:= :database.id database-id]
+ [:not= :fk-field.fk_target_field_id nil]
+ :fk-field.active]
+ :order-by [[:fk-field.id :desc]
+ [:pk-field.id :desc]]}))
diff --git a/src/metabase/permissions/db.clj b/src/metabase/permissions/db.clj
index 793580e141f..f91a558ad0c 100644
--- a/src/metabase/permissions/db.clj
+++ b/src/metabase/permissions/db.clj
@@ -3,6 +3,7 @@
additional logic, so the rest of the module only touches `toucan2.core` for model definitions, hydration methods,
and transactions."
(:require
+ [metabase.app-db.core :as mdb]
[metabase.permissions.schema :as permissions.schema]
[metabase.premium-features.core :as premium-features]
[metabase.settings.core :as setting]
@@ -260,6 +261,15 @@
[rows]
(t2/insert! :model/Permissions rows))
+(defn permission-objects-for-user
+ "The Permissions objects granted, via group membership, to the User with `user-id`."
+ [user-id]
+ (map :object (mdb/query {:select [:p.object]
+ :from [[:permissions_group_membership :pgm]]
+ :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]
+ [:permissions :p] [:= :p.group_id :pg.id]]
+ :where [:= :pgm.user_id user-id]})))
+
;;; --------------------------------------------- PermissionsGroup ---------------------------------------------
(defn magic-group
@@ -343,6 +353,19 @@
[group-id changes]
(t2/update! :model/PermissionsGroup group-id changes))
+(defn group-member-counts
+ "A map of PermissionsGroup ID to number of active members in the group. Groups with no active members have no
+ entry."
+ []
+ (let [results (mdb/query {:select [[:pgm.group_id :group_id] [[:count :pgm.id] :members]]
+ :from [[:permissions_group_membership :pgm]]
+ :left-join [[:core_user :user] [:= :pgm.user_id :user.id]]
+ :where [:= :user.is_active true]
+ :group-by [:pgm.group_id]})]
+ (zipmap
+ (map :group_id results)
+ (map :members results))))
+
;;; ---------------------------------------- PermissionsGroupMembership ----------------------------------------
(defn- insert-group-memberships-from-mapping-query
diff --git a/src/metabase/permissions/models/permissions_group.clj b/src/metabase/permissions/models/permissions_group.clj
index 11cee32a358..4334f36db0c 100644
--- a/src/metabase/permissions/models/permissions_group.clj
+++ b/src/metabase/permissions/models/permissions_group.clj
@@ -199,15 +199,7 @@
"Return a map of `PermissionsGroup` ID -> number of members in the group. (This doesn't include entries for empty
groups.)"
[]
- (let [results (mdb/query
- {:select [[:pgm.group_id :group_id] [[:count :pgm.id] :members]]
- :from [[:permissions_group_membership :pgm]]
- :left-join [[:core_user :user] [:= :pgm.user_id :user.id]]
- :where [:= :user.is_active true]
- :group-by [:pgm.group_id]})]
- (zipmap
- (map :group_id results)
- (map :members results))))
+ (permissions.db/group-member-counts))
(methodical/defmethod t2/batched-hydrate [:model/PermissionsGroup :member_count]
"Efficiently add `:member_count` to PermissionGroups."
diff --git a/src/metabase/permissions/user.clj b/src/metabase/permissions/user.clj
index f557c3116d1..26adf7dcd79 100644
--- a/src/metabase/permissions/user.clj
+++ b/src/metabase/permissions/user.clj
@@ -1,6 +1,6 @@
(ns metabase.permissions.user
(:require
- [metabase.app-db.core :as app-db]
+ [metabase.permissions.db :as permissions.db]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.path :as permissions.path]
[metabase.permissions.published-tables :as published-tables]
@@ -34,11 +34,7 @@
(map permissions.path/collection-readwrite-path ((requiring-resolve 'metabase.collections.models.collection/collections-in-namespace)
:transforms))))
;; include the other Perms entries for any Group this User is in (1 DB Call)
- (map :object (app-db/query {:select [:p.object]
- :from [[:permissions_group_membership :pgm]]
- :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]
- [:permissions :p] [:= :p.group_id :pg.id]]
- :where [:= :pgm.user_id user-id]})))))))
+ (permissions.db/permission-objects-for-user user-id))))))
(defn query-creation-capabilities
"Returns a map with `:can-create-queries` and `:can-create-native-queries` for the given user,
diff --git a/src/metabase/pulse/db.clj b/src/metabase/pulse/db.clj
index 21f61fd80a9..05ce83cf346 100644
--- a/src/metabase/pulse/db.clj
+++ b/src/metabase/pulse/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the pulse module. Every function here is a direct Toucan 2 call with no
additional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."
(:require
+ [metabase.app-db.core :as app-db]
[toucan2.core :as t2]))
(defn card
@@ -30,6 +31,24 @@
[dashboard-id card-ids]
(t2/select-fn->pk :card_id :model/DashboardCard :dashboard_id dashboard-id :card_id [:in card-ids]))
+(defn pulse-card-pairs-for-dashboard
+ "Distinct `:pulse-id`/`:card-id` pairs for the Pulses (subscriptions) attached to the Dashboard with
+ `dashboard-id`."
+ [dashboard-id]
+ (app-db/query {:select-distinct [[:p.id :pulse-id] [:pc.card_id :card-id]]
+ :from [[:pulse :p]]
+ :left-join [[:pulse_card :pc] [:= :p.id :pc.pulse_id]]
+ :where [:= :p.dashboard_id dashboard-id]}))
+
+(defn dashboard-card-ids-for-dashboard
+ "The distinct, non-nil Card ids shown by the DashboardCards of the Dashboard with `dashboard-id`."
+ [dashboard-id]
+ (map :card_id (app-db/query {:select-distinct [:dc.card_id]
+ :from [[:report_dashboardcard :dc]]
+ :where [:and
+ [:= :dc.dashboard_id dashboard-id]
+ [:not= :dc.card_id nil]]})))
+
(defn channel
"The Channel with `channel-id`, or nil."
[channel-id]
diff --git a/src/metabase/pulse/events/dashboard_subscription.clj b/src/metabase/pulse/events/dashboard_subscription.clj
index 987a08b5bff..421afde59e9 100644
--- a/src/metabase/pulse/events/dashboard_subscription.clj
+++ b/src/metabase/pulse/events/dashboard_subscription.clj
@@ -1,7 +1,6 @@
(ns metabase.pulse.events.dashboard-subscription
(:require
[clojure.set :as set]
- [metabase.app-db.core :as app-db]
[metabase.events.core :as events]
[metabase.pulse.db :as pulse.db]
[metabase.pulse.models.pulse :as models.pulse]
@@ -17,20 +16,9 @@
"Updates the pulses' names and collection IDs, and syncs the PulseCards"
[_ {dashboard :object}]
(let [dashboard-id (u/the-id dashboard)
- affected (app-db/query
- {:select-distinct [[:p.id :pulse-id] [:pc.card_id :card-id]]
- :from [[:pulse :p]]
- :left-join [[:pulse_card :pc] [:= :p.id :pc.pulse_id]]
- :where [:= :p.dashboard_id dashboard-id]})]
+ affected (pulse.db/pulse-card-pairs-for-dashboard dashboard-id)]
(when-let [pulse-ids (seq (distinct (map :pulse-id affected)))]
- (let [correct-card-ids (->> (app-db/query
- {:select-distinct [:dc.card_id]
- :from [[:report_dashboardcard :dc]]
- :where [:and
- [:= :dc.dashboard_id dashboard-id]
- [:not= :dc.card_id nil]]})
- (map :card_id)
- set)
+ (let [correct-card-ids (set (pulse.db/dashboard-card-ids-for-dashboard dashboard-id))
stale-card-ids (->> affected
(keep :card-id)
set)
diff --git a/src/metabase/query_processor/db.clj b/src/metabase/query_processor/db.clj
index 8f869b80120..d3d83f47ee6 100644
--- a/src/metabase/query_processor/db.clj
+++ b/src/metabase/query_processor/db.clj
@@ -3,6 +3,7 @@
additional logic, so the rest of the module never talks to `toucan2.core` itself."
(:require
[java-time.api :as t]
+ [metabase.app-db.core :as app-db]
^{:clj-kondo/ignore [:discouraged-namespace]}
[toucan2.core :as t2]))
@@ -71,3 +72,12 @@
"The `:id`, `:database_id`, and `:card_schema` of the Cards with `card-ids`."
[card-ids]
(t2/select [:model/Card :id :database_id :card_schema] :id [:in card-ids]))
+
+(defn upsert-cache-entry!
+ "Insert or update the QueryCache entry for `query-hash`, setting `:results` to `results` and `:updated_at` to
+ `timestamp`, and clearing `:refresh_started_at`."
+ [query-hash timestamp results]
+ (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}
+ (constantly {:updated_at timestamp
+ :results results
+ :refresh_started_at nil})))
diff --git a/src/metabase/query_processor/middleware/cache_backend/db.clj b/src/metabase/query_processor/middleware/cache_backend/db.clj
index 53b3a917dda..8ec65cf916b 100644
--- a/src/metabase/query_processor/middleware/cache_backend/db.clj
+++ b/src/metabase/query_processor/middleware/cache_backend/db.clj
@@ -1,7 +1,6 @@
(ns metabase.query-processor.middleware.cache-backend.db
(:require
[java-time.api :as t]
- [metabase.app-db.core :as app-db]
[metabase.premium-features.core :refer [defenterprise]]
[metabase.query-processor.db :as query-processor.db]
[metabase.query-processor.middleware.cache-backend.interface :as i]
@@ -119,10 +118,7 @@
(let [final-results (encryption/maybe-encrypt-for-stream results)
timestamp (t/offset-date-time)]
(try
- (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}
- (constantly {:updated_at timestamp
- :results final-results
- :refresh_started_at nil}))
+ (query-processor.db/upsert-cache-entry! query-hash timestamp final-results)
(catch Throwable e
(log/errorf "Error saving query results to cache: %s" (ex-message e))))
nil))
diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.clj
index c6bbc16f97e..baeec9605c4 100644
--- a/src/metabase/search/db.clj
+++ b/src/metabase/search/db.clj
@@ -3,6 +3,7 @@
additional logic, so no other namespace in the module runs a query itself (connection and transaction handling still use `toucan2.core`)."
(:require
[honey.sql.helpers :as sql.helpers]
+ [metabase.app-db.core :as mdb]
[metabase.search.appdb.specialization.api :as specialization]
[toucan2.core :as t2]))
@@ -11,6 +12,24 @@
[query]
(t2/query query))
+(defn spec-index-reducible-rows
+ "A reducible of the rows matching the Honey SQL `query` built from a search model's spec by
+ `metabase.search.ingestion`."
+ [query]
+ (mdb/streaming-reducible-query query))
+
+(defn in-place-model-set-rows
+ "The rows matching the Honey SQL `query` built by `metabase.search.in-place.legacy` to find the distinct set of
+ models with results."
+ [query]
+ (mdb/query query))
+
+(defn in-place-search-reducible
+ "A reducible of the rows matching the full in-place search Honey SQL `query` built by
+ `metabase.search.in-place.legacy`."
+ [query]
+ (mdb/streaming-reducible-query query))
+
(defn scored-search-rows
"The rows matching the scored, filtered search Honey SQL `query` built by `metabase.search.appdb.core`."
[query]
diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.clj
index d967e19509a..80b3124d7a1 100644
--- a/src/metabase/search/in_place/legacy.clj
+++ b/src/metabase/search/in_place/legacy.clj
@@ -10,6 +10,7 @@
[metabase.search.config
:as search.config
:refer [SearchContext SearchableModel]]
+ [metabase.search.db :as search.db]
[metabase.search.engine :as search.engine]
[metabase.search.filter :as search.filter]
[metabase.search.in-place.filter :as search.in-place.filter]
@@ -702,7 +703,7 @@
(cond-> {:select [:*]
:from [[^:allow-subquery {:union-all nested-queries} :dummy_alias]]}
(seq ctes) (assoc :with ctes)))]
- (into #{} (map :model) (some-> query mdb/query))))
+ (into #{} (map :model) (some-> query search.db/in-place-model-set-rows))))
(mu/defn full-search-query
"Postgres 9 is not happy with the type munging it needs to do to make the union-all degenerate down to a trivial case
@@ -735,7 +736,7 @@
(defn- results
[search-ctx]
(let [search-query (full-search-query search-ctx)]
- (mdb/streaming-reducible-query search-query)))
+ (search.db/in-place-search-reducible search-query)))
(defmethod search.engine/results
:search.engine/in-place
diff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.clj
index 8d5c5d94c95..fc47d77df90 100644
--- a/src/metabase/search/ingestion.clj
+++ b/src/metabase/search/ingestion.clj
@@ -203,7 +203,7 @@
;; joined side has its own integrity violations. Downstream upserts hit a unique constraint on
;; (model, model_id), so dedup here at the streaming boundary — bounded by per-model row count.
(->> (spec-index-query-where search-model where-clause)
- mdb/streaming-reducible-query
+ search.db/spec-index-reducible-rows
(eduction (comp (map #(assoc % :model search-model))
(m/distinct-by (juxt :id :model))))))
diff --git a/src/metabase/settings/db.clj b/src/metabase/settings/db.clj
index 1aab48c9714..e16ff494605 100644
--- a/src/metabase/settings/db.clj
+++ b/src/metabase/settings/db.clj
@@ -2,6 +2,7 @@
"Application database queries for the settings module. Every function here is a direct Toucan 2 call with no
additional logic, so the rest of the module only touches `toucan2.core` for model definitions."
(:require
+ [metabase.app-db.core :as mdb]
[toucan2.core :as t2]))
(defn setting-value
@@ -41,6 +42,11 @@
(t2/insert! (t2/table-name (t2/resolve-model :model/Setting))
:key setting-key, :value value, :value_with_aad value-with-aad))
+(defn current-timestamp-string
+ "The application DB's own current timestamp, as a string."
+ []
+ (mdb/current-timestamp-string (mdb/db-type)))
+
(defn update-user-settings!
"Store `settings-json` as the user-local settings of the User with `user-id`."
[user-id settings-json]
diff --git a/src/metabase/settings/models/setting/cache.clj b/src/metabase/settings/models/setting/cache.clj
index faf5913e670..a98badf5d1b 100644
--- a/src/metabase/settings/models/setting/cache.clj
+++ b/src/metabase/settings/models/setting/cache.clj
@@ -75,7 +75,7 @@
(log/debug "Updating value of settings-last-updated in DB...")
;; Written raw, not through `:model/Setting`, so that `value` gets the plaintext timestamp a version predating
;; `value_with_aad` compares in SQL. `value_with_aad` is encrypted under the marker's AAD like any other setting's.
- (let [value (mdb/current-timestamp-string (mdb/db-type))
+ (let [value (settings.db/current-timestamp-string)
value-with-aad (encryption/maybe-encrypt value {:aad (mdb.setting/setting-aad settings-last-updated-key)})]
;; attempt to UPDATE the existing row. If no row exists, `t2/update!` will return 0...
(or (pos? (settings.db/update-raw-setting-row! settings-last-updated-key value value-with-aad))
diff --git a/src/metabase/sync/analyze/data_sensitivity.clj b/src/metabase/sync/analyze/data_sensitivity.clj
index 55503fdfa3d..da8b7de2e38 100644
--- a/src/metabase/sync/analyze/data_sensitivity.clj
+++ b/src/metabase/sync/analyze/data_sensitivity.clj
@@ -8,6 +8,7 @@
change can reach fields that already carry a category."
(:require
[metabase.analyze.core :as analyze]
+ [metabase.sync.db :as sync.db]
[metabase.sync.interface :as i]
[metabase.sync.settings :as sync.settings]
[metabase.sync.util :as sync-util]
@@ -34,11 +35,6 @@
[:force? {:optional true} [:maybe :boolean]]
[:ignore-setting? {:optional true} [:maybe :boolean]]])
-(defn- selection-clause [force?]
- (if force?
- [:or [:= :data_sensitivity nil] [:= :data_sensitivity "PUBLIC"]]
- [:= :data_sensitivity nil]))
-
(defn- skip? [ignore-setting?]
(and (not ignore-setting?)
(not (sync.settings/data-sensitivity-scan-enabled))))
@@ -46,13 +42,7 @@
(mu/defn- fields-to-scan :- [:sequential i/FieldInstance]
[table :- i/TableInstance
force?]
- (t2/select :model/Field
- {:where [:and
- [:= :table_id (u/the-id table)]
- [:= :active true]
- [:not= :visibility_type "retired"]
- (selection-clause force?)]
- :order-by [[:id :asc]]}))
+ (sync.db/fields-to-scan-for-data-sensitivity (u/the-id table) force?))
(mu/defn- classify-and-save!
"Returns the inferred category, nil when no rule matched (in which case `:PUBLIC` was written), or the Exception."
@@ -60,7 +50,7 @@
table :- i/TableInstance]
(sync-util/with-error-handling (format "Error classifying data sensitivity for %s" (sync-util/name-for-logging field))
(let [category (analyze/infer-data-sensitivity field {:name (:name table) :entity_type (:entity_type table)})]
- (t2/update! :model/Field (u/the-id field) {:data_sensitivity (or category :PUBLIC)})
+ (sync.db/update-field-data-sensitivity! (u/the-id field) (or category :PUBLIC))
category)))
(mu/defn scan-table! :- Stats
@@ -88,17 +78,7 @@
(mu/defn- table-ids-with-unscanned-fields :- [:maybe [:set pos-int?]]
[database :- i/DatabaseInstance
force?]
- (t2/select-fn-set :table_id :model/Field
- {:select [[:metabase_field.table_id :table_id]]
- :from [:metabase_field]
- :join [[:metabase_table] [:= :metabase_field.table_id :metabase_table.id]]
- :where [:and
- [:= :metabase_table.db_id (u/the-id database)]
- [:= :metabase_table.active true]
- [:= :metabase_field.active true]
- [:not= :metabase_field.visibility_type "retired"]
- (selection-clause force?)]
- :group-by [:metabase_field.table_id]}))
+ (sync.db/table-ids-with-fields-to-scan-for-data-sensitivity (u/the-id database) force?))
(mu/defn scan-fields-for-db! :- Stats
"Label every unscanned Field in every active table of `database`. `log-fn` is accepted for parity with the other
@@ -115,35 +95,16 @@
(map #(scan-table! % :force? force? :ignore-setting? true)))
(partial merge-with +)
zero-stats
- (t2/reducible-select :model/Table
- :id [:in table-ids]
- {:order-by [[:schema :asc] [:name :asc]]}))))))
-
-(defn- scope-clause [database-or-table]
- (case (t2/model database-or-table)
- :model/Table [:= :table_id (u/the-id database-or-table)]
- :model/Database [:in :table_id ^:allow-subquery {:select [:id]
- :from [:metabase_table]
- :where [:= :db_id (u/the-id database-or-table)]}]))
-
-(def ^:private classifier-label-clause
- "A non-null `data_sensitivity` with no value in the `FieldUserSettings` mirror was written by the classifier."
- [:and
- [:not= :data_sensitivity nil]
- [:not [:exists ^:allow-subquery {:select [1]
- :from [[:metabase_field_user_settings :s]]
- :where [:and
- [:= :s.field_id :metabase_field.id]
- [:not= :s.data_sensitivity nil]]}]]])
+ (sync.db/tables-by-schema-and-name-reducible table-ids))))))
(mu/defn reset-data-sensitivity! :- :int
"REPL entry point: clear every classifier-written `data_sensitivity` in a Database or a single Table so the next
scan recomputes them, including fields that carry a category and would otherwise never be reselected. Labels with
a value in the `FieldUserSettings` mirror are human-set and untouched. Returns the number of fields cleared."
[database-or-table :- [:or i/DatabaseInstance i/TableInstance]]
- (let [n (t2/query-one {:update :metabase_field
- :set {:data_sensitivity nil}
- :where [:and (scope-clause database-or-table) classifier-label-clause]})]
+ (let [n (case (t2/model database-or-table)
+ :model/Table (sync.db/reset-classifier-data-sensitivity-for-table! (u/the-id database-or-table))
+ :model/Database (sync.db/reset-classifier-data-sensitivity-for-database! (u/the-id database-or-table)))]
(log/infof "Data sensitivity reset %d classifier-written labels in %s" n (sync-util/name-for-logging database-or-table))
n))
diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.clj
index 08533d0159c..7d80eebe165 100644
--- a/src/metabase/sync/db.clj
+++ b/src/metabase/sync/db.clj
@@ -513,6 +513,85 @@
(t2/query-one (mark-fk-statement db-id fk-table-schema fk-table-name fk-column-name
pk-table-schema pk-table-name pk-column-name)))
+;;; ------------------------------------------ Field data sensitivity ------------------------------------------
+
+(defn- data-sensitivity-to-scan-clause
+ "Honey SQL clause matching Fields the data-sensitivity classifier still has to scan: unlabeled ones, plus those it
+ labeled `PUBLIC` when `rescan-public?`."
+ [rescan-public?]
+ (if rescan-public?
+ [:or [:= :data_sensitivity nil] [:= :data_sensitivity "PUBLIC"]]
+ [:= :data_sensitivity nil]))
+
+(defn fields-to-scan-for-data-sensitivity
+ "The active, non-retired Fields of the Table with `table-id` that the data-sensitivity classifier still has to scan
+ (see [[data-sensitivity-to-scan-clause]]), ordered by ID."
+ [table-id rescan-public?]
+ (t2/select :model/Field
+ {:where [:and
+ [:= :table_id table-id]
+ [:= :active true]
+ [:not= :visibility_type "retired"]
+ (data-sensitivity-to-scan-clause rescan-public?)]
+ :order-by [[:id :asc]]}))
+
+(defn table-ids-with-fields-to-scan-for-data-sensitivity
+ "The IDs of the active Tables of the Database with `database-id` that have active, non-retired Fields the
+ data-sensitivity classifier still has to scan (see [[data-sensitivity-to-scan-clause]])."
+ [database-id rescan-public?]
+ (t2/select-fn-set :table_id :model/Field
+ {:select [[:metabase_field.table_id :table_id]]
+ :from [:metabase_field]
+ :join [[:metabase_table] [:= :metabase_field.table_id :metabase_table.id]]
+ :where [:and
+ [:= :metabase_table.db_id database-id]
+ [:= :metabase_table.active true]
+ [:= :metabase_field.active true]
+ [:not= :metabase_field.visibility_type "retired"]
+ (data-sensitivity-to-scan-clause rescan-public?)]
+ :group-by [:metabase_field.table_id]}))
+
+(defn tables-by-schema-and-name-reducible
+ "Reducible Tables with `table-ids`, ordered by schema and name."
+ [table-ids]
+ (t2/reducible-select :model/Table :id [:in table-ids] {:order-by [[:schema :asc] [:name :asc]]}))
+
+(defn update-field-data-sensitivity!
+ "Set the `data_sensitivity` of the Field with `field-id` to `data-sensitivity`."
+ [field-id data-sensitivity]
+ (t2/update! :model/Field field-id {:data_sensitivity data-sensitivity}))
+
+(def ^:private classifier-data-sensitivity-clause
+ "Honey SQL clause matching Fields whose non-null `data_sensitivity` has no value in the `FieldUserSettings` mirror,
+ i.e. was written by the data-sensitivity classifier rather than a user."
+ [:and
+ [:not= :data_sensitivity nil]
+ [:not [:exists ^:allow-subquery {:select [1]
+ :from [[:metabase_field_user_settings :s]]
+ :where [:and
+ [:= :s.field_id :metabase_field.id]
+ [:not= :s.data_sensitivity nil]]}]]])
+
+(defn reset-classifier-data-sensitivity-for-table!
+ "Clear the classifier-written `data_sensitivity` (see [[classifier-data-sensitivity-clause]]) of the Fields of the
+ Table with `table-id`. Returns the number of Fields cleared."
+ [table-id]
+ (t2/query-one {:update :metabase_field
+ :set {:data_sensitivity nil}
+ :where [:and [:= :table_id table-id] classifier-data-sensitivity-clause]}))
+
+(defn reset-classifier-data-sensitivity-for-database!
+ "Clear the classifier-written `data_sensitivity` (see [[classifier-data-sensitivity-clause]]) of the Fields of every
+ Table of the Database with `database-id`. Returns the number of Fields cleared."
+ [database-id]
+ (t2/query-one {:update :metabase_field
+ :set {:data_sensitivity nil}
+ :where [:and
+ [:in :table_id ^:allow-subquery {:select [:id]
+ :from [:metabase_table]
+ :where [:= :db_id database-id]}]
+ classifier-data-sensitivity-clause]}))
+
;;; ---------------------------------------------- FieldValues ----------------------------------------------
(defn field-values-exist?
diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.clj
index 9d9f2ae2968..03d8a7c382e 100644
--- a/src/metabase/transforms/models/transform_run.clj
+++ b/src/metabase/transforms/models/transform_run.clj
@@ -38,7 +38,7 @@
(boolean (when-let [transform-id (:transform_id instance)]
(mi/can-read? :model/Transform transform-id)))))
([_model pk]
- (when-let [run (t2/select-one :model/TransformRun :id pk)]
+ (when-let [run (transforms.db/run pk)]
(mi/can-read? run))))
(mi/define-simple-hydration-method add-transform-runs
diff --git a/src/metabase/warehouse_schema/db.clj b/src/metabase/warehouse_schema/db.clj
index 8de4de6e3ab..5eff57a4c7a 100644
--- a/src/metabase/warehouse_schema/db.clj
+++ b/src/metabase/warehouse_schema/db.clj
@@ -225,6 +225,15 @@
[field-values-id changes]
(t2/update! :model/FieldValues field-values-id changes))
+(defn find-or-insert-full-field-values!
+ "The full FieldValues of the Field with `field-id`, inserting one with `has-more-values`, `values`, and no
+ `human_readable_values` if none exists yet."
+ [field-id has-more-values values]
+ (app-db/select-or-insert! :model/FieldValues {:field_id field-id, :type :full}
+ (constantly {:has_more_values has-more-values
+ :values values
+ :human_readable_values nil})))
+
(defn touch-field-values!
"Stamp `last_used_at` on the FieldValues with `field-values-id`."
[field-values-id]
diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.clj
index 06094e891a4..81ab782c284 100644
--- a/src/metabase/warehouse_schema/models/field_values.clj
+++ b/src/metabase/warehouse_schema/models/field_values.clj
@@ -27,7 +27,6 @@
[java-time.api :as t]
[medley.core :as m]
[metabase.analyze.core :as analyze]
- [metabase.app-db.core :as app-db]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.schema.metadata :as lib.schema.metadata]
@@ -478,10 +477,7 @@
(nil? existing-fv)
(do
(log/debugf "Storing FieldValues for Field %s..." field-name)
- (app-db/select-or-insert! :model/FieldValues {:field_id (u/the-id field), :type :full}
- (constantly {:has_more_values has-more-values
- :values values
- :human_readable_values nil}))
+ (warehouse-schema.db/find-or-insert-full-field-values! (u/the-id field) has-more-values values)
::fv-created)
;; if existing FieldValues won't change, skip it
diff --git a/src/metabase/warehouse_schema_rest/api/field.clj b/src/metabase/warehouse_schema_rest/api/field.clj
index 8868c9e17f2..bc094549bc4 100644
--- a/src/metabase/warehouse_schema_rest/api/field.clj
+++ b/src/metabase/warehouse_schema_rest/api/field.clj
@@ -3,7 +3,6 @@
[metabase.analytics.core :as analytics]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
- [metabase.app-db.core :as app-db]
[metabase.events.core :as events]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.models.interface :as mi]
@@ -323,11 +322,11 @@
[400 (str "You can only update the human readable values of a mapped values of a Field whose value of "
"`has_field_values` is `list` or whose 'base_type' is 'type/Boolean'.")])
(let [human-readable-values? (validate-human-readable-pairs value-pairs)
- update-map {:values (map first value-pairs)
- :human_readable_values (when human-readable-values?
- (map second value-pairs))}
- updated-pk (app-db/update-or-insert! :model/FieldValues {:field_id (u/the-id field), :type :full}
- (constantly update-map))]
+ values (map first value-pairs)
+ human-readable-values (when human-readable-values?
+ (map second value-pairs))
+ updated-pk (warehouse-schema-rest.db/update-or-insert-full-field-values!
+ (u/the-id field) values human-readable-values)]
(api/check-500 (pos? updated-pk))))
{:status :success})
diff --git a/src/metabase/warehouse_schema_rest/db.clj b/src/metabase/warehouse_schema_rest/db.clj
index 1f6e6a554b0..5fea0e2944c 100644
--- a/src/metabase/warehouse_schema_rest/db.clj
+++ b/src/metabase/warehouse_schema_rest/db.clj
@@ -157,3 +157,11 @@
"Delete the FieldValues of the Fields with `field-ids`."
[field-ids]
(t2/delete! (t2/table-name :model/FieldValues) :field_id [:in field-ids]))
+
+(defn update-or-insert-full-field-values!
+ "Update the full FieldValues of the Field with `field-id` to have `values` and `human-readable-values`, inserting
+ one if none exists yet. Returns the number of rows affected."
+ [field-id values human-readable-values]
+ (app-db/update-or-insert! :model/FieldValues {:field_id field-id, :type :full}
+ (constantly {:values values
+ :human_readable_values human-readable-values})))
diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.clj
index 20b1fc2d65f..55af4910680 100644
--- a/src/metabase/warehouses_rest/api.clj
+++ b/src/metabase/warehouses_rest/api.clj
@@ -6,7 +6,6 @@
[metabase.analytics.core :as analytics]
[metabase.api.common :as api]
[metabase.api.macros :as api.macros]
- [metabase.app-db.core :as mdb]
[metabase.classloader.core :as classloader]
[metabase.config.core :as config]
[metabase.database-routing.core :as database-routing]
@@ -504,52 +503,6 @@
:include-editable-data-model? include_editable_data_model
:exclude-uneditable-details? exclude_uneditable_details}))
-(def ^:private database-usage-models
- "List of models that are used to report usage on a database."
- [:question :dataset :metric :segment :transform]) ; TODO -- rename `:dataset` to `:model`?
-
-(defmulti ^:private database-usage-query
- "Query that will returns the number of `model` that use the database with id `database-id`.
- The query must returns a scalar, and the method could return `nil` in case no query is available."
- {:arglists '([model database-id])}
- (fn [model _database-id] (keyword model)))
-
-(defn- card-query
- [db-id model type-str]
- ^:allow-subquery {:select [[:%count.* model]]
- :from [:report_card]
- :where [:and
- [:= :database_id db-id]
- [:= :type type-str]]})
-
-(defmethod database-usage-query :question
- [_ db-id]
- (card-query db-id :question "question"))
-
-(defmethod database-usage-query :dataset
- [_ db-id]
- (card-query db-id :dataset "model"))
-
-(defmethod database-usage-query :metric
- [_ db-id]
- (card-query db-id :metric "metric"))
-
-(defmethod database-usage-query :segment
- [_ db-id]
- ^:allow-subquery {:select [[:%count.* :segment]]
- :from [:segment]
- :where [:in :table_id ^:allow-subquery {:select [:id]
- :from [:metabase_table]
- :where [:= :db_id db-id]}]})
-
-(defmethod database-usage-query :transform
- [_ db-id]
- ^:allow-subquery {:select [[:%count.* :transform]]
- :from [:transform]
- :where [:or
- [:= :source_database_id db-id]
- [:= :target_db_id db-id]]})
-
;; TODO (Cam 10/28/25) -- fix this endpoint route to use kebab-case for consistency with the rest of our REST API
;;
;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to
@@ -564,12 +517,7 @@
[:id ms/PositiveInt]]]
(api/check-superuser)
(check-database-exists id)
- (first (mdb/query
- {:select [:*]
- :from (for [model database-usage-models
- :let [query (database-usage-query model id)]
- :when query]
- [query model])})))
+ (first (warehouses-rest.db/database-usage-counts id)))
;;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------
diff --git a/src/metabase/warehouses_rest/db.clj b/src/metabase/warehouses_rest/db.clj
index d8f2fefbf23..b741540f44f 100644
--- a/src/metabase/warehouses_rest/db.clj
+++ b/src/metabase/warehouses_rest/db.clj
@@ -287,3 +287,33 @@
"The ids of the Collections named `collection-name`, or nil."
[collection-name]
(t2/select-pks-set :model/Collection :name collection-name))
+
+(defn- card-usage-count-subquery
+ [database-id model type-str]
+ ^:allow-subquery {:select [[:%count.* model]]
+ :from [:report_card]
+ :where [:and
+ [:= :database_id database-id]
+ [:= :type type-str]]})
+
+(defn database-usage-counts
+ "A single row with the count of Questions (`:question`), Models (`:dataset`), Metrics (`:metric`), Segments
+ (`:segment`), and Transforms (`:transform`) that use the Database with `database-id`."
+ [database-id]
+ (mdb/query
+ {:select [:*]
+ :from [[(card-usage-count-subquery database-id :question "question") :question]
+ [(card-usage-count-subquery database-id :dataset "model") :dataset]
+ [(card-usage-count-subquery database-id :metric "metric") :metric]
+ [^:allow-subquery {:select [[:%count.* :segment]]
+ :from [:segment]
+ :where [:in :table_id ^:allow-subquery {:select [:id]
+ :from [:metabase_table]
+ :where [:= :db_id database-id]}]}
+ :segment]
+ [^:allow-subquery {:select [[:%count.* :transform]]
+ :from [:transform]
+ :where [:or
+ [:= :source_database_id database-id]
+ [:= :target_db_id database-id]]}
+ :transform]]}))