The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add kondo rule keeping Toucan 2 query calls in module db namespaces (#82099)
Alexander Polyankin15106f17fda5
diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.ednindex 80424f7a1db..65998f56f9c 100644--- a/.clj-kondo/config.edn+++ b/.clj-kondo/config.edn@@ -72,6 +72,7 @@:metabase/namespace-name {:level :warning}:metabase/no-jsqlparser-imports {:level :warning}:metabase/prefer-with-dynamic-fn-redefs {:level :warning}+ :metabase/t2-query-namespace {:level :warning}:metabase/toucan-model-ns {:level :warning}:metabase/require-shape-checker {:level :warning}:metabase/tests-must-live-in-known-modules {:level :warning}@@ -633,6 +634,45 @@metabase.lib.schema.mbql-clause/define-tuple-mbql-clause hooks.metabase.lib.schema.mbql-clause/define-mbql-clausemethodical.core/defmethod hooks.metabase.toucan.table-name/lint-defmethodmethodical.macros/defmethod hooks.metabase.toucan.table-name/lint-defmethod+ metabase.app-db.core/query hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.core/streaming-reducible-query hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.core/select-or-insert! hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.core/update-or-insert! hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.core/current-timestamp-string hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.query/query hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.query/streaming-reducible-query hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.query/select-or-insert! hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.query/update-or-insert! hooks.metabase.toucan.db-ns/lint-query-call+ metabase.app-db.query/current-timestamp-string hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/count hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/delete! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/exists? hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/insert! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/insert-returning-instance! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/insert-returning-instances! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/insert-returning-pk! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/insert-returning-pks! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/query hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/query-one hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/reducible-query hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/reducible-select hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/reducible-update hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/reducible-update-returning-pks hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/save! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-fn->fn hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-fn->pk hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-fn-reducible hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-fn-set hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-fn-vec hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-one hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-one-fn hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-one-pk hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-pk->fn hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-pks-set hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/select-pks-vec hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/update! hooks.metabase.toucan.db-ns/lint-query-call+ toucan2.core/update-returning-pks! hooks.metabase.toucan.db-ns/lint-query-callmetabase.lib.test-util.macros/$ids hooks.metabase.test.data/$idsmetabase.lib.test-util.macros/mbql-query hooks.metabase.test.data/mbql-querymetabase.lib.test-util.macros/mbql-5-query hooks.metabase.test.data/mbql-query@@ -841,7 +881,13 @@{:pattern "^metabase(?:((?:-enterprise\\.sandbox)?\\.query-processor)|(?:\\.driver\\.sql(?!server)(?!ite))).*-test$":name general-qp-and-driver-test-namespaces}{:pattern "(?:^mage.*)"- :name mage-namespaces}]+ :name mage-namespaces}+ ;;+ ;; the app-db module is the application database layer itself (connections, migrations, encryption), so it gets to+ ;; run Toucan 2 queries anywhere.+ ;;+ {:pattern "^metabase\\.app-db\\..*"+ :name app-db-namespaces}]:config-in-ns{test-namespaces@@ -851,6 +897,7 @@:private-call {:level :off}:metabase/defsetting-must-specify-export {:level :off}:metabase/defsetting-namespace {:level :off}+ :metabase/t2-query-namespace {:level :off}:metabase/test-helpers-use-non-thread-safe-functions {:level :warning} ; more config in `.clj-kondo/config/test-helpers/config.edn`:discouraged-var@@ -858,6 +905,9 @@metabase.lib.convert/->legacy-MBQL {:level :off}metabase.lib.core/->legacy-MBQL {:level :off}}}}+ app-db-namespaces+ {:linters {:metabase/t2-query-namespace {:level :off}}}+;; .cljc tests run in both Clojure and ClojureScript — `with-dynamic-fn-redefs` is JVM-only,;; so leave these alone.metabase.lib.aggregation-testdiff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 0749f5ba677..dfab7040c23 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -61,6 +61,7 @@:metabase/defsetting-must-specify-export 1:metabase/defsetting-namespace 1:metabase/prefer-with-dynamic-fn-redefs 3+ :metabase/t2-query-namespace 2:missing-docstring 1:private-call 1:redundant-ignore 1diff --git a/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj b/.clj-kondo/src/hooks/metabase/toucan/db_ns.cljnew file mode 100644index 00000000000..db2296c7a69--- /dev/null+++ b/.clj-kondo/src/hooks/metabase/toucan/db_ns.clj@@ -0,0 +1,35 @@+(ns hooks.metabase.toucan.db-ns+ "Lint that application database query calls -- Toucan 2's `t2/select`, `t2/query`, `t2/insert!`, `t2/update!`,+ `t2/delete!` and friends, plus the `metabase.app-db.core` wrappers around them such as `mdb/query` and+ `mdb/update-or-insert!` -- live in a module's `db` namespace, i.e. `metabase[-enterprise].<module>.db` (or `metabase.driver.<driver>.db` for+ driver modules). Every other namespace in a module goes through those functions instead of talking to the+ application database directly.++ Registered as an `:analyze-call` hook on each of those functions in `.clj-kondo/config.edn`. The hook+ returns its input unchanged so Kondo's normal analysis of the call (arity, var usage) still runs."+ (:require+ [clj-kondo.hooks-api :as hooks]))++(defn- db-namespace? [ns-sym]+ (boolean (re-matches #"^metabase(?:-enterprise)?\.(?:driver\.)?[^.]+\.db$" (name ns-sym))))++(defn- test-file?+ "Whether `filename` is in a test source tree. Test namespaces are exempt through the `test-namespaces` group in+ `config.edn`, but helper namespaces like `metabase.sso.test-helpers` don't match that group's pattern."+ [filename]+ (boolean (and filename (re-find #"(?:^|/)test/" filename))))++(defn lint-query-call+ "Register a `:metabase/t2-query-namespace` finding when a Toucan 2 query call appears outside a `<module>.db`+ namespace."+ [{:keys [node ns filename] :as input}]+ (when (and ns+ (not (db-namespace? ns))+ (not (test-file? filename)))+ (let [fn-node (first (:children node))]+ (hooks/reg-finding!+ (assoc (meta fn-node)+ :message (format "Application database query calls like `%s` must live in metabase[-enterprise].<module>.db (or metabase.driver.<driver>.db) namespaces"+ (hooks/sexpr fn-node))+ :type :metabase/t2-query-namespace))))+ input)diff --git a/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.cljnew file mode 100644index 00000000000..2109ad98d1d--- /dev/null+++ b/.clj-kondo/test/hooks/metabase/toucan/db_ns_test.clj@@ -0,0 +1,50 @@+(ns hooks.metabase.toucan.db-ns-test+ (:require+ [clj-kondo.hooks-api :as hooks]+ [clj-kondo.impl.utils]+ [clojure.test :refer :all]+ [hooks.metabase.toucan.db-ns :as toucan.db-ns]))++(defn- lint-query-call [form ns-sym & [filename]]+ (binding [clj-kondo.impl.utils/*ctx* {:config {:linters {:metabase/t2-query-namespace {:level :warning}}}+ :ignores (atom nil)+ :findings (atom [])+ :namespaces (atom {})}]+ (let [input {:node (hooks/parse-string (pr-str form))+ :ns ns-sym+ :filename (or filename "src/metabase/foo/bar.clj")}+ output (toucan.db-ns/lint-query-call input)]+ (is (identical? (:node input) (:node output))+ "the hook must return the node unchanged so Kondo's normal analysis still runs")+ @(:findings clj-kondo.impl.utils/*ctx*))))++(deftest ^:parallel t2-query-must-live-in-db-namespace-test+ (testing "a query call outside a db namespace is flagged"+ (is (=? [{:type :metabase/t2-query-namespace+ :message #".*`t2/select-one`.*"}]+ (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase.queries.models.card))))+ (testing "metabase.<module>.db is allowed"+ (is (empty? (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase.queries.db))))+ (testing "metabase-enterprise.<module>.db is allowed"+ (is (empty? (lint-query-call '(t2/select-one :model/Card :id 1) 'metabase-enterprise.sandbox.db))))+ (testing "metabase.driver.<driver>.db is allowed for driver modules"+ (is (empty? (lint-query-call '(t2/select-one :model/Database :id 1) 'metabase.driver.bigquery-cloud-sdk.db))))+ (testing "a nested db namespace is not a module db namespace"+ (is (=? [{:type :metabase/t2-query-namespace}]+ (lint-query-call '(t2/query {:select [:*]}) 'metabase.queries.models.db))))+ (testing "a namespace merely ending in db is not a module db namespace"+ (is (=? [{:type :metabase/t2-query-namespace}]+ (lint-query-call '(t2/delete! :model/Card :id 1) 'metabase.queries.dashboards-db))))+ (testing "files in a test source tree are exempt even when the namespace name doesn't look like a test"+ (is (empty? (lint-query-call '(t2/update! :model/User 1 {:sso_source nil}) 'metabase.sso.test-helpers+ "test/metabase/sso/test_helpers.clj")))+ (is (empty? (lint-query-call '(t2/select :model/Card) 'metabase-enterprise.remote-sync.test-helpers+ "/repo/enterprise/backend/test/metabase_enterprise/remote_sync/test_helpers.clj"))))+ (testing "app-db wrappers around Toucan 2 are reported the same way"+ (is (=? [{:type :metabase/t2-query-namespace+ :message #".*`mdb/update-or-insert!`.*"}]+ (lint-query-call '(mdb/update-or-insert! :model/Foo {:id 1}) 'metabase.foo.models.foo))))+ (testing "fully-qualified calls are reported by their written name"+ (is (=? [{:type :metabase/t2-query-namespace+ :message #".*`toucan2.core/insert!`.*"}]+ (lint-query-call '(toucan2.core/insert! :model/Card {}) 'metabase.queries.models.card)))))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.cljindex 64ac847cb58..c62cf4faa9f 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj@@ -608,6 +608,40 @@[entity-type entity-id](t2/delete! :model/DependencyStatus :entity_type entity-type :entity_id entity-id))+(defn mark-dependency-status-stale!+ "Mark the DependencyStatus of `entity-type` `entity-id` as stale for dependency recalculation, creating it if it+ doesn't exist. Resets retry state so previously-failed entities get a fresh chance."+ [entity-type entity-id]+ (mdb/update-or-insert!+ :model/DependencyStatus+ {:entity_type entity-type :entity_id entity-id}+ (fn [existing]+ (if existing+ {:stale true :fail_count 0 :next_retry_at nil :terminal false}+ {:stale true :dependency_analysis_version 0}))))++(defn upsert-dependency-status!+ "Upsert the DependencyStatus of `entity-type` `entity-id`, setting stale=false, `dependency_analysis_version` to+ `current-version`, and clearing any failure state."+ [entity-type entity-id current-version]+ (mdb/update-or-insert!+ :model/DependencyStatus+ {:entity_type entity-type :entity_id entity-id}+ (fn [_existing]+ {:dependency_analysis_version current-version+ :stale false+ :fail_count 0+ :next_retry_at nil})))++(defn record-dependency-status-failure!+ "Upsert the DependencyStatus of `entity-type` `entity-id`, creating it if needed. `update-fn` receives the+ existing DependencyStatus row (or nil if none exists) and must return the columns to set."+ [entity-type entity-id update-fn]+ (mdb/update-or-insert!+ :model/DependencyStatus+ {:entity_type entity-type :entity_id entity-id}+ update-fn))+(defn pending-retry-exists?"Whether a non-terminal DependencyStatus is waiting for a retry."[]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.cljindex 8f28846bfda..0513a1660d2 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency_status.clj@@ -4,7 +4,6 @@[metabase-enterprise.dependencies.db :as dependencies.db][metabase-enterprise.dependencies.dependency-types :as deps.dependency-types][metabase-enterprise.dependencies.models.dependency :as models.dependency]- [metabase.app-db.core :as app-db][metabase.models.interface :as mi][methodical.core :as methodical][toucan2.core :as t2]))@@ -19,30 +18,17 @@(defn mark-stale!"Mark entities of `entity-type` with ids in `entity-ids` as stale for dependency recalculation.Creates entries if they don't exist, or sets stale=true if they do.- Resets retry state so previously-failed entities get a fresh chance.- Uses [[app-db/update-or-insert!]] for cross-database atomicity."+ Resets retry state so previously-failed entities get a fresh chance."[entity-type entity-ids](doseq [id entity-ids]- (app-db/update-or-insert!- :model/DependencyStatus- {:entity_type entity-type :entity_id id}- (fn [existing]- (if existing- {:stale true :fail_count 0 :next_retry_at nil :terminal false}- {:stale true :dependency_analysis_version 0})))))+ (dependencies.db/mark-dependency-status-stale! entity-type id)))(defn upsert-status!"Upsert a dependency_status entry, setting stale=false, version to current,- and clearing any failure state. Uses [[app-db/update-or-insert!]] for cross-database atomicity."+ and clearing any failure state."[entity-type entity-id]- (app-db/update-or-insert!- :model/DependencyStatus- {:entity_type entity-type :entity_id entity-id}- (fn [_existing]- {:dependency_analysis_version models.dependency/current-dependency-analysis-version- :stale false- :fail_count 0- :next_retry_at nil})))+ (dependencies.db/upsert-dependency-status!+ entity-type entity-id models.dependency/current-dependency-analysis-version))(defmulti hydrate-for-deps"Hydrate a batch of instances with data needed for dependency calculation.@@ -87,12 +73,10 @@Increments fail_count and sets next_retry_at based on exponential backoff.If max retries exceeded, marks the entity as terminal.Creates the entry if it doesn't exist, since entities with no row yet are exactly the ones- [[instances-for-dependency-calculation]] picks up first.- Uses [[app-db/update-or-insert!]] for cross-database atomicity."+ [[instances-for-dependency-calculation]] picks up first."[entity-type entity-id max-retries delay-minutes]- (app-db/update-or-insert!- :model/DependencyStatus- {:entity_type entity-type :entity_id entity-id}+ (dependencies.db/record-dependency-status-failure!+ entity-type entity-id(fn [existing];; An inserted row takes the column defaults for `stale` (false) and `dependency_analysis_version` (0). 0 is;; below `current-dependency-analysis-version`, which is what keeps the entity eligible for the retry oncediff --git a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj b/enterprise/backend/src/metabase_enterprise/sandbox/db.cljindex ef42ea87ba3..d0269500662 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/db.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/db.clj@@ -2,6 +2,7 @@"Application database queries for the sandbox module. Every function here is a direct Toucan 2 call with noadditional logic, so the rest of the module only touches `toucan2.core` for model definitions."(:require+ [metabase.app-db.core :as mdb][toucan2.core :as t2]))(defn sandbox@@ -54,6 +55,23 @@(when db-id [:= :t.db_id db-id])(when excluded-db-id [:not [:= :t.db_id excluded-db-id]])]}))+(defn candidate-sandboxes-for-groups-and-databases+ "The `:id`, `:group_id`, `:table_id`, `:db_id`, and `:schema` of the Sandboxes of the groups with `group-ids` on+ Tables of the Databases with `db-ids`."+ [group-ids db-ids]+ (mdb/query+ {:select [[:sandboxes.id :id]+ [:sandboxes.group_id :group_id]+ [:sandboxes.table_id :table_id]+ [:table.db_id :db_id]+ [:table.schema :schema]]+ :from [[:sandboxes]]+ :left-join [[:metabase_table :table]+ [:= :sandboxes.table_id :table.id]]+ :where [:and+ [:in :sandboxes.group_id group-ids]+ [:in :table.db_id db-ids]]}))+(defn insert-sandbox!"Insert `sandbox` and return the new instance."[sandbox]diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.cljindex 4ec307fdd4e..1bd34e9c233 100644--- a/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj+++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/permissions/delete_sandboxes.clj@@ -1,7 +1,6 @@(ns metabase-enterprise.sandbox.models.permissions.delete-sandboxes(:require[metabase-enterprise.sandbox.db :as sandbox.db]- [metabase.app-db.core :as app-db][metabase.premium-features.core :refer [defenterprise]][metabase.util.i18n :refer [tru]][metabase.util.log :as log]))@@ -45,18 +44,8 @@:when (contains? db-changes :view-data)]db-id))](when (and (seq all-group-ids) (seq all-db-ids))- (let [candidate-sandboxes (app-db/query- {:select [[:sandboxes.id :id]- [:sandboxes.group_id :group_id]- [:sandboxes.table_id :table_id]- [:table.db_id :db_id]- [:table.schema :schema]]- :from [[:sandboxes]]- :left-join [[:metabase_table :table]- [:= :sandboxes.table_id :table.id]]- :where [:and- [:in :sandboxes.group_id all-group-ids]- [:in :table.db_id all-db-ids]]})+ (let [candidate-sandboxes (sandbox.db/candidate-sandboxes-for-groups-and-databases+ all-group-ids all-db-ids)ids-to-delete (into #{}(comp (filter (partial should-delete-sandbox? changes))(map :id))diff --git a/enterprise/backend/src/metabase_enterprise/security_center/db.clj b/enterprise/backend/src/metabase_enterprise/security_center/db.cljindex 2e65ee16425..49e174667ab 100644--- a/enterprise/backend/src/metabase_enterprise/security_center/db.clj+++ b/enterprise/backend/src/metabase_enterprise/security_center/db.clj@@ -2,6 +2,7 @@"Application database queries for the security-center module. Every function here is a direct Toucan 2 call with noadditional logic, so the rest of the module only touches `toucan2.core` for model definitions and hydration methods."(:require+ [metabase.app-db.core :as mdb][toucan2.core :as t2]))(defn advisory@@ -14,6 +15,12 @@[advisory-id](t2/select-one :model/SecurityAdvisory :advisory_id advisory-id))+(defn max-advisory-updated-at+ "The latest `updated_at` across every SecurityAdvisory, or nil when there are none."+ []+ (:updated_at (t2/query-one {:select [[[:max :updated_at] :updated_at]]+ :from [:security_advisory]})))+(defn advisories-newest-first"Every SecurityAdvisory, newest first."[]@@ -44,6 +51,18 @@[id changes](t2/update! :model/SecurityAdvisory id changes))+(defn upsert-advisory!+ "Insert or update a SecurityAdvisory by `:advisory_id`. On insert, `:match_status` starts as `:unknown` until the+ matching engine evaluates it. On update, sets `advisory`'s columns, leaving `:match_status`, `:last_evaluated_at`,+ and acknowledgement fields (which `advisory` doesn't include) untouched."+ [advisory]+ (mdb/update-or-insert! :model/SecurityAdvisory+ {:advisory_id (:advisory_id advisory)}+ (fn [existing]+ (if existing+ advisory+ (assoc advisory :match_status :unknown)))))+(defn user-summaries-by-id"A map of ID to the ID, names, and email of the Users with `user-ids`."[user-ids]diff --git a/enterprise/backend/src/metabase_enterprise/security_center/fetch.clj b/enterprise/backend/src/metabase_enterprise/security_center/fetch.cljindex f1577e2c73a..d1ce73ba771 100644--- a/enterprise/backend/src/metabase_enterprise/security_center/fetch.clj+++ b/enterprise/backend/src/metabase_enterprise/security_center/fetch.clj@@ -5,8 +5,8 @@[clojure.edn :as edn][clojure.set :as set][java-time.api :as t]+ [metabase-enterprise.security-center.db :as security-center.db][metabase-enterprise.security-center.schema :as security-center.schema]- [metabase.app-db.core :as mdb][metabase.config.core :as config][metabase.premium-features.core :as premium-features][metabase.util.json :as json]@@ -60,11 +60,7 @@(defn- latest-updated-at"Return the maximum `updated_at` across all advisories as an ISO-8601 string, or nil if none exist."[]- (some-> (mdb/query {:select [[[:max :updated_at]]]- :from [:security_advisory]})- first- vals- first+ (some-> (security-center.db/max-advisory-updated-at)t/instantt/format))@@ -132,12 +128,7 @@On insert, match_status starts as :unknown until the matching engine evaluates it.On update, merges new data but preserves :match_status, :last_evaluated_at, and acknowledgement fields."[advisory]- (mdb/update-or-insert! :model/SecurityAdvisory- {:advisory_id (:advisory_id advisory)}- (fn [existing]- (if existing- advisory- (assoc advisory :match_status :unknown)))))+ (security-center.db/upsert-advisory! advisory))(defn sync-advisories!"Fetch advisories from the MetaStore and upsert into the appdb."diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/db.cljindex b588dc6c6d9..bfc4e2d6e89 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/db.clj@@ -2,6 +2,7 @@"Application database queries for the transforms-python module. Every function here is a direct Toucan 2 call with noadditional logic, so the rest of the module only touches `toucan2.core` for model definitions and hydration methods."(:require+ [metabase.app-db.core :as mdb][toucan2.core :as t2]))(defn table-database-ids@@ -39,6 +40,13 @@[path](t2/select-one :model/PythonLibrary :path path))+(defn upsert-python-library-source!+ "Insert or update the PythonLibrary at `path`, setting its source to `source`. Returns the ID of the row."+ [path source]+ (mdb/update-or-insert! :model/PythonLibrary+ {:path path}+ (constantly {:path path :source source})))+(defn library-sources-by-path"A map of path to source for every PythonLibrary."[]diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.cljindex e8529c101bf..712d50ba319 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/models/python_library.clj@@ -2,7 +2,6 @@(:require[metabase-enterprise.transforms-python.db :as transforms-python.db][metabase.api.common :as api]- [metabase.app-db.core :as app-db][metabase.events.core :as events][metabase.models.interface :as mi][metabase.models.serialization :as serdes]@@ -78,9 +77,7 @@[path source](let [normalized-path (normalize-path path)](validate-path! normalized-path)- (let [id (app-db/update-or-insert! :model/PythonLibrary- {:path normalized-path}- (constantly {:path normalized-path :source source}))]+ (let [id (transforms-python.db/upsert-python-library-source! normalized-path source)](transforms-python.db/python-library id))));;; ------------------------------------------------- Serialization --------------------------------------------------diff --git a/src/metabase/app_db/db.clj b/src/metabase/app_db/db.cljindex a30a97dc329..27a6966cd4d 100644--- a/src/metabase/app_db/db.clj+++ b/src/metabase/app_db/db.clj@@ -6,8 +6,37 @@first, and while rows are being re-encrypted a setting row can be plaintext under a key, or ciphertext under a keynot yet in effect, which that model's strict read rejects."(:require+ [honey.sql :as sql]+ [metabase.util.honey-sql-2 :as h2x][toucan2.core :as t2]))+(defn current-timestamp-string+ "The application DB's own current timestamp, as a string, for app DB type `db-type`."+ ^String [db-type]+ ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)+ (let [cast-form (h2x/cast (if (= db-type :mysql) :char :text) (h2x/current-datetime-honeysql-form db-type))]+ (:timestamp (t2/query-one {:select [[cast-form :timestamp]]}))))++;;; ------------------------------------------------ Liquibase ------------------------------------------------++(defn changelog-by-id+ "The Liquibase changelog row with `changelog-id` in the app DB of type `db-type`, or nil."+ [db-type changelog-id]+ (let [table-name (case db-type+ (:postgres :h2) "databasechangelog"+ :mysql "DATABASECHANGELOG")]+ (t2/query-one [(format "select * from %s where id = ?" table-name) changelog-id])))++(defn changelog-ids+ "The ids among `changelog-ids` still present in the Liquibase changelog table `changelog-table-name`, read on+ `conn`."+ [conn changelog-table-name changelog-ids]+ (map :id (t2/query conn (sql/format {:select [:id]+ :from [(keyword changelog-table-name)]+ :where [:in :id changelog-ids]}))))++;;; ------------------------------------------------ Settings ------------------------------------------------+(def ^:private unmigrated-settings-where[:and [:= :value_with_aad nil] [:not= :value nil] [:not= :value ""]])diff --git a/src/metabase/app_db/liquibase.clj b/src/metabase/app_db/liquibase.cljindex 4afbb4f4467..c81e9675953 100644--- a/src/metabase/app_db/liquibase.clj+++ b/src/metabase/app_db/liquibase.clj@@ -3,10 +3,9 @@(:require[clojure.java.jdbc :as jdbc][clojure.string :as str]- [honey.sql :as sql]- [honey.sql.helpers :as sql.helpers][metabase.app-db.connection :as mdb.connection][metabase.app-db.custom-migrations]+ [metabase.app-db.db :as app-db.db][metabase.app-db.liquibase.h2 :as liquibase.h2][metabase.app-db.liquibase.mysql :as liquibase.mysql][metabase.classloader.core :as classloader]@@ -16,8 +15,7 @@[metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.schema :as ms]- [toucan2.connection :as t2.conn]- [toucan2.core :as t2])+ [toucan2.connection :as t2.conn])(:import(java.io StringWriter)(java.sql Connection)@@ -176,10 +174,7 @@(defn changelog-by-id"Return the changelog row value for the given `changelog-id`."[app-db changelog-id]- (let [table-name (case (:db-type app-db)- (:postgres :h2) "databasechangelog"- :mysql "DATABASECHANGELOG")]- (t2/query-one (format "select * from %s where id = '%s'" table-name changelog-id))))+ (app-db.db/changelog-by-id (:db-type app-db) changelog-id))(defn migrations-sql"Return a string of SQL containing the DDL statements needed to perform unrun `liquibase` migrations, custom migrations will be ignored."@@ -597,11 +592,7 @@(.getChangeLogParameters liquibase)changelogchange-listener))- (let [remaining-query (-> (sql.helpers/select :id)- (sql.helpers/from (keyword (changelog-table-name liquibase)))- (sql.helpers/where [:in :id ids-to-drop]))- formatted-sql (sql/format remaining-query)- remaining-ids (map :id (t2/query conn formatted-sql))]+ (let [remaining-ids (app-db.db/changelog-ids conn (changelog-table-name liquibase) ids-to-drop)](when (seq remaining-ids)(log/warnf "The following changesets were not rolled back. Likely because %s: %s"(if (seq @error-ids)diff --git a/src/metabase/app_db/query.clj b/src/metabase/app_db/query.cljindex 5580cdef999..c389528700d 100644--- a/src/metabase/app_db/query.clj+++ b/src/metabase/app_db/query.clj@@ -22,9 +22,9 @@(:require[clojure.string :as str][honey.sql :as sql]+ [metabase.app-db.db :as app-db.db][metabase.app-db.format :as app-db.format][metabase.util :as u]- [metabase.util.honey-sql-2 :as h2x][metabase.util.malli :as mu][metabase.util.malli.schema :as ms][toucan2.core :as t2]@@ -121,9 +121,7 @@"The application DB's own current timestamp, as a string. Read from the DB rather than from this machine's clock,for the `settings-last-updated` marker that instances compare against each other."^String [db-type]- ;; for MySQL, cast(current_timestamp AS char); for H2 & Postgres, cast(current_timestamp AS text)- (let [cast-form (h2x/cast (if (= db-type :mysql) :char :text) (h2x/current-datetime-honeysql-form db-type))]- (-> (t2/query-one {:select [[cast-form :timestamp]]}) :timestamp)))+ (app-db.db/current-timestamp-string db-type));;; TODO -- we should mark this deprecated and tell people to use [[toucan2.core/query]] directly instead(defn querydiff --git a/src/metabase/cache/db.clj b/src/metabase/cache/db.cljindex 6a71ee4bf1e..310c211d47d 100644--- a/src/metabase/cache/db.clj+++ b/src/metabase/cache/db.clj@@ -2,6 +2,7 @@"Application database queries for the cache module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [metabase.app-db.core :as app-db][toucan2.core :as t2]))(defn database-with-ids@@ -103,6 +104,12 @@[model model-id](t2/select-one :model/CacheConfig :model model :model_id model-id {:for :update}))+(defn upsert-cache-config!+ "Insert or replace the CacheConfig for `model` and `model-id` with `data`."+ [model model-id data]+ (app-db/update-or-insert! :model/CacheConfig {:model model :model_id model-id}+ (constantly data)))+(defn cache-configs-for"The CacheConfigs for `model` and `model-ids`."[model model-ids]diff --git a/src/metabase/cache/models/cache_config.clj b/src/metabase/cache/models/cache_config.cljindex 30d08e1dcab..4e2c419ecf9 100644--- a/src/metabase/cache/models/cache_config.clj+++ b/src/metabase/cache/models/cache_config.clj@@ -3,7 +3,6 @@(:require[java-time.api :as t][medley.core :as m]- [metabase.app-db.core :as app-db][metabase.cache.db :as cache.db][metabase.events.core :as events][metabase.models.interface :as mi]@@ -168,8 +167,7 @@(t2/with-transaction [_tx](let [data (config->row config)current (cache.db/lock-cache-config model model_id)]- (u/prog1 (app-db/update-or-insert! :model/CacheConfig {:model model :model_id model_id}- (constantly data))+ (u/prog1 (cache.db/upsert-cache-config! model model_id data)(audit-caching-change! user-id <> current data)))))(defn delete!diff --git a/src/metabase/channel/db.clj b/src/metabase/channel/db.cljindex 59af45f697a..6ab6a68e83b 100644--- a/src/metabase/channel/db.clj+++ b/src/metabase/channel/db.clj@@ -2,6 +2,7 @@"Application database queries for the channel module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [metabase.app-db.core :as app-db][toucan2.core :as t2]))(defn channels@@ -56,6 +57,20 @@[:= :is_active true][:in :id user-ids]]}))+(defn user-ids-with-permission+ "The ids of the Users belonging to a PermissionsGroup that holds `permission-path`."+ [permission-path]+ (app-db/query {:select [:pgm.user_id]+ :from [[:permissions_group_membership :pgm]]+ :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]]+ :where [:exists ^:allow-subquery+ {:select [1]+ :from [[:permissions :p]]+ :where [:and+ [:= :p.group_id :pg.id]+ [:= :p.object permission-path]]}]+ :group-by [:pgm.user_id]}))+(defn delete-pulse-channels-for-channel!"Delete the PulseChannels of the Channel with `channel-id`."[channel-id]diff --git a/src/metabase/channel/email/messages.clj b/src/metabase/channel/email/messages.cljindex 7e0500758a3..6fe6243d4c3 100644--- a/src/metabase/channel/email/messages.clj+++ b/src/metabase/channel/email/messages.clj@@ -8,7 +8,6 @@[clojure.string :as str][java-time.api :as t][medley.core :as m]- [metabase.app-db.core :as app-db][metabase.appearance.core :as appearance][metabase.channel.db :as channel.db][metabase.channel.email :as email]@@ -249,19 +248,7 @@[database-id](let [monitoring (perms/application-perms-path :monitoring)user-ids-with-monitoring (when (premium-features/enable-advanced-permissions?)- (->> {:select [:pgm.user_id]- :from [[:permissions_group_membership :pgm]]- :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]]- :where [:and- [:exists ^:allow-subquery- {:select [1]- :from [[:permissions :p]]- :where [:and- [:= :p.group_id :pg.id]- [:= :p.object monitoring]]}]]- :group-by [:pgm.user_id]}- app-db/query- (mapv :user_id)))+ (mapv :user_id (channel.db/user-ids-with-permission monitoring)))user-ids (filter#(perms/user-has-permission-for-database? % :perms/manage-database :yes database-id)user-ids-with-monitoring)]diff --git a/src/metabase/collections_rest/api.clj b/src/metabase/collections_rest/api.cljindex 776e57298de..ca53d356797 100644--- a/src/metabase/collections_rest/api.clj+++ b/src/metabase/collections_rest/api.clj@@ -1114,7 +1114,7 @@[rows rows-query offset](or (some-> rows first :total_count)(when (pos? (or offset 0))- (some-> (mdb/query (assoc rows-query :limit 1)) first :total_count))+ (some-> (collections-rest.db/collection-children-rows (assoc rows-query :limit 1)) first :total_count))0))(defn- collection-children*@@ -1157,7 +1157,7 @@:limit (if (zero? limit) 1 limit):offset offset))rows (tracing/with-span :db-app "db-app.collection-items-query" {:collection/id (:id collection)}- (mdb/query limit-query))+ (collections-rest.db/collection-children-rows limit-query))res {:total (total-count rows rows-query offset):data (if (= limit 0)[]@@ -1219,7 +1219,7 @@:permission-level (if archived? :write :read):include-trash-collection? archived?}row (first- (mdb/query+ (collections-rest.db/collection-filter-metadata-rows{:with [[:visible_collection_ids (collection/visible-collection-query viz-config)]]:select (vec(for [model candidates]diff --git a/src/metabase/collections_rest/db.clj b/src/metabase/collections_rest/db.cljindex 08112c56896..a5a294c4345 100644--- a/src/metabase/collections_rest/db.clj+++ b/src/metabase/collections_rest/db.clj@@ -2,6 +2,7 @@"Application database queries for the collections REST module. Every function here is a direct Toucan 2 call with noadditional logic, so the rest of the module only touches `toucan2.core` for hydration."(:require+ [metabase.app-db.core :as app-db][metabase.collections.models.collection :as collection][toucan2.core :as t2]))@@ -165,3 +166,17 @@"The Cards in the Collection with `collection-id`."[collection-id](t2/select :model/Card :collection_id collection-id))++(defn collection-children-rows+ "The rows matching the collection-children Honey SQL `query`, built by `metabase.collections-rest.api` from the+ per-model item queries for a Collection's paginated child listing. Follows the same exception as+ `metabase.search.db` for spec-driven Honey SQL that can't be reduced to plain-data parameters."+ [query]+ (app-db/query query))++(defn collection-filter-metadata-rows+ "The rows matching the collection-filter-metadata Honey SQL `query`, built by `metabase.collections-rest.api` to+ probe which item models have at least one visible child in a Collection. Follows the same exception as+ `metabase.search.db` for spec-driven Honey SQL that can't be reduced to plain-data parameters."+ [query]+ (app-db/query query))diff --git a/src/metabase/content_verification/db.clj b/src/metabase/content_verification/db.cljindex ea3c42a9c36..e7048ba4c4a 100644--- a/src/metabase/content_verification/db.clj+++ b/src/metabase/content_verification/db.clj@@ -2,6 +2,7 @@"Application database queries for the content verification module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [metabase.app-db.core :as app-db][toucan2.core :as t2]))(defn moderation-reviews-for-items@@ -17,6 +18,16 @@[user-ids](t2/select :model/User :id [:in user-ids]))+(defn moderation-review-ids-for-item+ "The ids of the ModerationReviews of the item with `item-id` and `item-type`, newest first."+ [item-id item-type]+ (app-db/query {:select [:id]+ :from [:moderation_review]+ :where [:and+ [:= :moderated_item_id item-id]+ [:= :moderated_item_type item-type]]+ :order-by [[:id :desc]]}))+(defn most-recent-moderation-review-statuses"The item id, item type, and status of the most recent ModerationReviews of the items with `item-types` and`item-ids`, newest first."diff --git a/src/metabase/content_verification/models/moderation_review.clj b/src/metabase/content_verification/models/moderation_review.cljindex 9245dc32633..b9b023b90dc 100644--- a/src/metabase/content_verification/models/moderation_review.clj+++ b/src/metabase/content_verification/models/moderation_review.clj@@ -1,6 +1,5 @@(ns metabase.content-verification.models.moderation-review(:require- [metabase.app-db.core :as app-db][metabase.content-verification.db :as content-verification.db][metabase.content-verification.impl :as moderation][metabase.models.interface :as mi]@@ -49,18 +48,12 @@ensures there are one fewer than that so you can add afterwards."[item-id :- :intitem-type :- :string]- (let [ids (into #{} (comp (map :id)+ (let [;; cannot put the offset in this query as mysql doesn't play nice. It requires a limit as well which we do+ ;; not want to give. The offset is only 10 though so its not a huge savings and we run this on every entry so+ ;; the max number is 10, delete the extra, and insert a new one to arrive at 10 again, our invariant.+ ids (into #{} (comp (map :id)(drop (dec max-moderation-reviews)))- (app-db/query {:select [:id]- :from [:moderation_review]- :where [:and- [:= :moderated_item_id item-id]- [:= :moderated_item_type item-type]]- ;; cannot put the offset in this query as mysql doesn't play nice. It requires a limit- ;; as well which we do not want to give. The offset is only 10 though so its not a huge- ;; savings and we run this on every entry so the max number is 10, delete the extra,- ;; and insert a new one to arrive at 10 again, our invariant.- :order-by [[:id :desc]]}))]+ (content-verification.db/moderation-review-ids-for-item item-id item-type))](when (seq ids)(content-verification.db/delete-moderation-reviews! ids))))diff --git a/src/metabase/dashboards/db.clj b/src/metabase/dashboards/db.cljindex 0c0f47843a2..7b0c24afcaf 100644--- a/src/metabase/dashboards/db.clj+++ b/src/metabase/dashboards/db.clj@@ -40,6 +40,14 @@:dashboard_id [:in dashboard-ids]{:order-by [[:dashboard_id :asc] [:position :asc] [:id :asc]]}))+(defn dashboard-collection-authority-levels+ "The id and Collection `authority_level` of the Dashboards with `dashboard-ids`."+ [dashboard-ids]+ (mdb/query {:select [:dashboard.id :collection.authority_level]+ :from [[:report_dashboard :dashboard]]+ :left-join [[:collection :collection] [:= :collection.id :dashboard.collection_id]]+ :where [:in :dashboard.id dashboard-ids]}))+(defn dashcards-with-visible-cards-for-dashboards"The DashboardCards of the Dashboards with `dashboard-ids` whose Card is visible (unarchived, a dashboard questionnot archived by itself, or absent), with their Card's Collection authority level, in dashboard then creation order."@@ -138,6 +146,19 @@[dashcard-id](t2/select-one :model/DashboardCard :id dashcard-id))+(defn multi-cards-for-dashcard+ "The unarchived Cards added to the DashboardCard with `dashcard-id` as series (the 'add series' feature)."+ [dashcard-id]+ (mdb/query {:select [:newcard.*]+ :from [[:report_dashboardcard :dashcard]]+ :left-join [[:dashboardcard_series :dashcardseries]+ [:= :dashcard.id :dashcardseries.dashboardcard_id]+ [:report_card :newcard]+ [:= :dashcardseries.card_id :newcard.id]]+ :where [:and+ [:= :newcard.archived false]+ [:= :dashcard.id dashcard-id]]}))+(defn dashcards-by-ids"The DashboardCards with `dashcard-ids`."[dashcard-ids]diff --git a/src/metabase/dashboards/models/dashboard.clj b/src/metabase/dashboards/models/dashboard.cljindex bda35299c77..96d10850ae6 100644--- a/src/metabase/dashboards/models/dashboard.clj+++ b/src/metabase/dashboards/models/dashboard.clj@@ -3,7 +3,6 @@[clojure.set :as set][medley.core :as m][metabase.api.common :as api]- [metabase.app-db.core :as app-db][metabase.audit-app.core :as audit][metabase.collections.core :as collections][metabase.collections.models.collection :as collection]@@ -164,10 +163,8 @@(when (seq dashboards)(let [coll-id->level (into {}(map (juxt :id :authority_level))- (app-db/query {:select [:dashboard.id :collection.authority_level]- :from [[:report_dashboard :dashboard]]- :left-join [[:collection :collection] [:= :collection.id :dashboard.collection_id]]- :where [:in :dashboard.id (into #{} (map u/the-id) dashboards)]}))]+ (dashboards.db/dashboard-collection-authority-levels+ (into #{} (map u/the-id) dashboards)))](for [dashboard dashboards](assoc dashboard :collection_authority_level (get coll-id->level (u/the-id dashboard)))))))diff --git a/src/metabase/dashboards/models/dashboard_card.clj b/src/metabase/dashboards/models/dashboard_card.cljindex 7cbf17bc87a..26fb504af46 100644--- a/src/metabase/dashboards/models/dashboard_card.clj+++ b/src/metabase/dashboards/models/dashboard_card.clj@@ -2,7 +2,6 @@(:require[clojure.set :as set][medley.core :as m]- [metabase.app-db.core :as mdb][metabase.dashboards.db :as dashboards.db][metabase.models.interface :as mi][metabase.models.serialization :as serdes]@@ -159,15 +158,7 @@This is also different from having multiple series displayed on Line, Area, or Bar Questions."[dashcard]- (mdb/query {:select [:newcard.*]- :from [[:report_dashboardcard :dashcard]]- :left-join [[:dashboardcard_series :dashcardseries]- [:= :dashcard.id :dashcardseries.dashboardcard_id]- [:report_card :newcard]- [:= :dashcardseries.card_id :newcard.id]]- :where [:and- [:= :newcard.archived false]- [:= :dashcard.id (:id dashcard)]]}))+ (dashboards.db/multi-cards-for-dashcard (:id dashcard)))(defn update-dashboard-cards-series!"Batch update the DashboardCardSeries for multiple DashboardCards.diff --git a/src/metabase/dashboards_rest/api.clj b/src/metabase/dashboards_rest/api.cljindex ba3d288e5f9..a084f2ca5a6 100644--- a/src/metabase/dashboards_rest/api.clj+++ b/src/metabase/dashboards_rest/api.clj@@ -10,7 +10,6 @@[metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.api.macros :as api.macros]- [metabase.app-db.core :as app-db][metabase.channel.email.messages :as messages][metabase.channel.render.core :as channel.render][metabase.collections-rest.api :as api.collection]@@ -49,7 +48,6 @@[metabase.request.core :as request][metabase.revisions.core :as revisions][metabase.util :as u]- [metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [deferred-tru tru]][metabase.util.log :as log][metabase.util.malli :as mu]@@ -715,34 +713,7 @@;; Output should match the shape of api/collection/<:id|root>/items. There's a test that asserts that this remains;; the case, but if you change one, you'll want to change both.(let [dashboard (api/read-check :model/Dashboard id)- query (merge- {:select [:c.id :c.name :c.description :c.entity_id :c.collection_position :c.display :c.collection_preview- :last_used_at :c.collection_id :c.archived_directly :c.archived :c.database_id- :c.dashboard_id- [nil :location]- [(h2x/literal "card") :model]- [^:allow-subquery {:select [:status]- :from [:moderation_review]- :where [:and- [:= :moderated_item_type "card"]- [:= :moderated_item_id :c.id]- [:= :most_recent true]]- ;; limit 1 to ensure that there is only one result but this invariant should hold true, just- ;; protecting against potential bugs- :order-by [[:id :desc]]- :limit 1}- :moderated_status]]- :from [[:report_card :c]]- :where [:and- [:= :c.dashboard_id id]- [:exists ^:allow-subquery {:select 1- :from [[:report_dashboardcard :dc]]- :where [:and [:= :c.id :dc.card_id] [:= :c.dashboard_id :dc.dashboard_id]]}]- [:= :c.archived false]]}- (when (request/paged?)- {:limit (request/limit)- :offset (request/offset)}))- cards (app-db/query query)]+ cards (dashboards-rest.db/dashboard-item-cards id (request/paged?) (request/limit) (request/offset))]{:total (count cards):data (api.collection/post-process-rows {}(dashboards-rest.db/collection (:collection_id dashboard))diff --git a/src/metabase/dashboards_rest/db.clj b/src/metabase/dashboards_rest/db.cljindex 924e24dcb6c..2bc7b035766 100644--- a/src/metabase/dashboards_rest/db.clj+++ b/src/metabase/dashboards_rest/db.clj@@ -2,6 +2,8 @@"Application database queries for the dashboards REST module. Every function here is a direct Toucan 2 call with noadditional logic, so the rest of the module only touches `toucan2.core` for hydration."(:require+ [metabase.app-db.core :as app-db]+ [metabase.util.honey-sql-2 :as h2x][toucan2.core :as t2]))(defn dashboards@@ -70,6 +72,37 @@[dashboard-id](t2/select-fn-vec :card_id :model/DashboardCard :dashboard_id dashboard-id))+(defn dashboard-item-cards+ "The card items of the Dashboard with `dashboard-id`, for the `/:id/items` endpoint: id, name, description, entity+ id, collection position, display, collection preview, last-used-at, collection id, archived flags, database id,+ and moderated status. Paged by `limit`/`offset` when `paged?`."+ [dashboard-id paged? limit offset]+ (app-db/query+ (cond-> {:select [:c.id :c.name :c.description :c.entity_id :c.collection_position :c.display :c.collection_preview+ :last_used_at :c.collection_id :c.archived_directly :c.archived :c.database_id+ :c.dashboard_id+ [nil :location]+ [(h2x/literal "card") :model]+ [^:allow-subquery {:select [:status]+ :from [:moderation_review]+ :where [:and+ [:= :moderated_item_type "card"]+ [:= :moderated_item_id :c.id]+ [:= :most_recent true]]+ ;; limit 1 to ensure that there is only one result but this invariant should+ ;; hold true, just protecting against potential bugs+ :order-by [[:id :desc]]+ :limit 1}+ :moderated_status]]+ :from [[:report_card :c]]+ :where [:and+ [:= :c.dashboard_id dashboard-id]+ [:exists ^:allow-subquery {:select 1+ :from [[:report_dashboardcard :dc]]+ :where [:and [:= :c.id :dc.card_id] [:= :c.dashboard_id :dc.dashboard_id]]}]+ [:= :c.archived false]]}+ paged? (merge {:limit limit :offset offset}))))+(defn dashboard-series-card-ids"The Card ids of the DashboardCardSeries of the Dashboard with `dashboard-id`."[dashboard-id]diff --git a/src/metabase/mcp/db.clj b/src/metabase/mcp/db.cljindex 16227ccf50e..0a56168750b 100644--- a/src/metabase/mcp/db.clj+++ b/src/metabase/mcp/db.clj@@ -2,6 +2,8 @@"Application database queries for the MCP module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [metabase.app-db.core :as app-db]+ [metabase.session.core :as session][toucan2.core :as t2]))(defn insert-feedback!@@ -14,6 +16,20 @@[key-hashed](t2/select-one-fn :user_id :core_session :key_hashed key-hashed))+(defn get-or-create-core-session!+ "The `core_session` row for `key-hashed` and `user-id`, creating one with a freshly generated session id if none+ exists. Uses the raw `:core_session` table (not `:model/Session`) to bypass the after-insert hook, which would+ otherwise publish a spurious `:event/user-login` event."+ [key-hashed user-id]+ (app-db/select-or-insert!+ :core_session+ {:key_hashed key-hashed+ :user_id user-id}+ (fn []+ {:id (session/generate-session-id)+ :anti_csrf_token nil+ :created_at :%now})))+(defn insert-query-handle!"Insert the McpQueryHandle `row`."[row]diff --git a/src/metabase/mcp/session.clj b/src/metabase/mcp/session.cljindex c170d2e4416..facbef54544 100644--- a/src/metabase/mcp/session.clj+++ b/src/metabase/mcp/session.clj@@ -13,7 +13,6 @@TTL and is re-created on demand for query-handle lifecycle management."(:require[clojure.string :as str]- [metabase.app-db.core :as app-db][metabase.mcp.db :as mcp.db][metabase.mcp.models.mcp-query-handle][metabase.mcp.settings :as mcp.settings]@@ -291,17 +290,7 @@;; colliding rows arbitrarily. Cross-user UUID collisions are an unaddressed risk;; across the whole session model (cookie sessions included) — it belongs in the;; auth layer, not here, and no constraint shape at this call site can fix it.- ;;- ;; Raw :core_session (not :model/Session) to bypass the after-insert hook, which- ;; would publish spurious :event/user-login events.- (app-db/select-or-insert!- :core_session- {:key_hashed key-hashed- :user_id user-id}- (fn []- {:id (session/generate-session-id)- :anti_csrf_token nil- :created_at :%now}))))+ (mcp.db/get-or-create-core-session! key-hashed user-id)))(defn get-or-create-session-key!"Ensure a `core_session` exists for this MCP session and return its (plaintext)diff --git a/src/metabase/metabot/db.clj b/src/metabase/metabot/db.cljindex 1c1d0d4c040..cef75ebc65c 100644--- a/src/metabase/metabot/db.clj+++ b/src/metabase/metabot/db.clj@@ -234,6 +234,12 @@[conversation](t2/insert! :model/MetabotConversation conversation))+(defn upsert-conversation!+ "Insert or update the MetabotConversation with `conversation-id`. `update-fn` receives the existing row (or nil on+ insert) and must return the fields to write."+ [conversation-id update-fn]+ (mdb/update-or-insert! :model/MetabotConversation {:id conversation-id} update-fn))+(defn set-conversation-title-if-missing!"Set the title of the MetabotConversation with `conversation-id` if it has none."[conversation-id title]@@ -310,6 +316,29 @@[rows](t2/insert! :model/MetabotUsedTable rows))+;;; --------------------------------------------------- Feedback ---------------------------------------------------++(defn upsert-feedback!+ "Insert or update the MetabotFeedback row for the MetabotMessage with `message-id` and the User with+ `submitter-user-id`. `update-fn` receives the existing row (or nil on insert) and must return the fields to+ write."+ [message-id submitter-user-id update-fn]+ (mdb/update-or-insert! :model/MetabotFeedback+ {:message_id message-id :user_id submitter-user-id}+ update-fn))++(defn upsert-source-feedback!+ "Insert or update the MetabotSourceFeedback row for the MetabotMessage with `message-id`, the User with+ `submitter-user-id`, and the source with `source-id`/`source-type`. `update-fn` receives the existing row (or nil+ on insert) and must return the fields to write."+ [message-id submitter-user-id source-id source-type update-fn]+ (mdb/update-or-insert! :model/MetabotSourceFeedback+ {:message_id message-id+ :user_id submitter-user-id+ :source_id source-id+ :source_type source-type}+ update-fn))+;;; ------------------------------------------------ Databases ------------------------------------------------(defn database-summarydiff --git a/src/metabase/metabot/feedback.clj b/src/metabase/metabot/feedback.cljindex babd98c5c40..b85bcf43756 100644--- a/src/metabase/metabot/feedback.clj+++ b/src/metabase/metabot/feedback.clj@@ -2,7 +2,6 @@"Local persistence of Metabot feedback."(:require[metabase.api.common :as api]- [metabase.app-db.core :as app-db][metabase.metabot.db :as metabot.db][metabase.models.interface :as mi]))@@ -27,25 +26,19 @@(let [base-fields {:positive positive:issue_type issue_type:freeform_feedback freeform_feedback}]- (app-db/update-or-insert! :model/MetabotFeedback- {:message_id message-row-id :user_id submitter-user-id}- (fn [existing]- (cond-> base-fields- existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))+ (metabot.db/upsert-feedback! message-row-id submitter-user-id+ (fn [existing]+ (cond-> base-fields+ existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))(defn- upsert-source-feedback!"Insert or update the `metabot_source_feedback` row for one source, message, and submitter."[message-row-id submitter-user-id {:keys [positive source_id source_type]}]- (let [conditions {:message_id message-row-id- :user_id submitter-user-id- :source_id source_id- :source_type source_type}- base-fields {:positive positive}]- (app-db/update-or-insert! :model/MetabotSourceFeedback- conditions- (fn [existing]- (cond-> base-fields- existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))+ (let [base-fields {:positive positive}]+ (metabot.db/upsert-source-feedback! message-row-id submitter-user-id source_id source_type+ (fn [existing]+ (cond-> base-fields+ existing (assoc :updated_at (java.time.OffsetDateTime/now)))))))(defn persist-feedback!"Upsert a `metabot_feedback` row for the rated message and return thediff --git a/src/metabase/metabot/persistence.clj b/src/metabase/metabot/persistence.cljindex c3ea40eff11..dd0110dc441 100644--- a/src/metabase/metabot/persistence.clj+++ b/src/metabase/metabot/persistence.clj@@ -4,7 +4,6 @@[clojure.string :as str][metabase.analytics-interface.core :as analytics][metabase.api.common :as api]- [metabase.app-db.core :as app-db][metabase.llm.provider :as llm.provider][metabase.metabot.agent.memory :as memory][metabase.metabot.agent.streaming :as streaming]@@ -276,28 +275,29 @@(t2/with-transaction [_conn](when (seq delete-message-ids)(soft-delete-messages! {:id [:in delete-message-ids]} originator-id))- (app-db/update-or-insert! :model/MetabotConversation {:id conversation-id}- (fn [existing]- ;; `:user_id` is the originator — set on insert, never overwritten.- (cond-> {}- (nil? existing)- (assoc :user_id originator-id)- (and hostname (nil? (:embedding_hostname existing)))- (assoc :embedding_hostname hostname)- (and (:embedding_path pii-info) (nil? (:embedding_path existing)))- (assoc :embedding_path (:embedding_path pii-info))- (and (:user_agent pii-info) (nil? (:user_agent existing)))- (assoc :user_agent (:user_agent pii-info))- (and (:sanitized_user_agent pii-info) (nil? (:sanitized_user_agent existing)))- (assoc :sanitized_user_agent (:sanitized_user_agent pii-info))- (and (:ip_address pii-info) (nil? (:ip_address existing)))- (assoc :ip_address (:ip_address pii-info))- (and slack-team-id (nil? (:slack_team_id existing)))- (assoc :slack_team_id slack-team-id)- (and channel-id (nil? (:slack_channel_id existing)))- (assoc :slack_channel_id channel-id)- (and slack-thread-ts (nil? (:slack_thread_ts existing)))- (assoc :slack_thread_ts slack-thread-ts))))+ (metabot.db/upsert-conversation!+ conversation-id+ (fn [existing]+ ;; `:user_id` is the originator — set on insert, never overwritten.+ (cond-> {}+ (nil? existing)+ (assoc :user_id originator-id)+ (and hostname (nil? (:embedding_hostname existing)))+ (assoc :embedding_hostname hostname)+ (and (:embedding_path pii-info) (nil? (:embedding_path existing)))+ (assoc :embedding_path (:embedding_path pii-info))+ (and (:user_agent pii-info) (nil? (:user_agent existing)))+ (assoc :user_agent (:user_agent pii-info))+ (and (:sanitized_user_agent pii-info) (nil? (:sanitized_user_agent existing)))+ (assoc :sanitized_user_agent (:sanitized_user_agent pii-info))+ (and (:ip_address pii-info) (nil? (:ip_address existing)))+ (assoc :ip_address (:ip_address pii-info))+ (and slack-team-id (nil? (:slack_team_id existing)))+ (assoc :slack_team_id slack-team-id)+ (and channel-id (nil? (:slack_channel_id existing)))+ (assoc :slack_channel_id channel-id)+ (and slack-thread-ts (nil? (:slack_thread_ts existing)))+ (assoc :slack_thread_ts slack-thread-ts))))(metabot.db/insert-messages!(cond-> {:conversation_id conversation-id:data (schema.v2/check-message-data "metabot_message.data"diff --git a/src/metabase/osi/ai_context/api.clj b/src/metabase/osi/ai_context/api.cljindex 500ed939fc9..e700e71ab48 100644--- a/src/metabase/osi/ai_context/api.clj+++ b/src/metabase/osi/ai_context/api.clj@@ -4,7 +4,6 @@(:require[metabase.api.common :as api][metabase.api.macros :as api.macros]- [metabase.app-db.core :as app-db][metabase.entity-retrieval.core :as entity-retrieval][metabase.osi.db :as osi.db][metabase.osi.models.osi-ai-context :as osi-ai-context]@@ -110,10 +109,7 @@;; Upsert on the normalized (stored) key so re-posting a relabelled card updates its one row.;; update-or-insert! handles the compound key, the no-op re-PUT, and the concurrent-create race;; (savepoint + single retry) centrally.- (app-db/update-or-insert! :model/OsiAiContext- {:entity_type (entity-retrieval/normalize-entity-type entity-type)- :entity_local_id entity-local-id}- (constantly {:ai_context ai_context}))+ (osi.db/upsert-ai-context! (entity-retrieval/normalize-entity-type entity-type) entity-local-id ai_context)(get-entry entity-type entity-local-id))(api.macros/defendpoint :post "/reconcile"diff --git a/src/metabase/osi/db.clj b/src/metabase/osi/db.cljindex 0c6a4a45244..7671c8ae42a 100644--- a/src/metabase/osi/db.clj+++ b/src/metabase/osi/db.clj@@ -2,6 +2,7 @@"Application database queries for the OSI module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [metabase.app-db.core :as app-db][toucan2.core :as t2]))(defn ai-context@@ -31,3 +32,12 @@"Apply `changes` to the OsiAiContext of the entity with `entity-type` and `entity-local-id`."[entity-type entity-local-id changes](t2/update! :model/OsiAiContext :entity_type entity-type :entity_local_id entity-local-id changes))++(defn upsert-ai-context!+ "Insert or replace the `:ai_context` of the OsiAiContext of the entity with `entity-type` and `entity-local-id`+ with `ai-context`."+ [entity-type entity-local-id ai-context]+ (app-db/update-or-insert! :model/OsiAiContext+ {:entity_type entity-type+ :entity_local_id entity-local-id}+ (constantly {:ai_context ai-context})))diff --git a/src/metabase/parameters/chain_filter.clj b/src/metabase/parameters/chain_filter.cljindex 849eb2b03c9..dbd98dbbb42 100644--- a/src/metabase/parameters/chain_filter.clj+++ b/src/metabase/parameters/chain_filter.clj@@ -201,22 +201,7 @@(u/minutes->ms 5))(defn- database-fk-relationships* [database-id enable-reverse-joins?]- (let [rows (mdb/query {:select [[:fk-field.id :f1]- [:fk-table.id :t1]- [:pk-field.id :f2]- [:pk-field.table_id :t2]]- :from [[:metabase_field :fk-field]]- :left-join [[:metabase_table :fk-table] [:and [:= :fk-field.table_id :fk-table.id]- :fk-table.active]- [:metabase_database :database] [:= :fk-table.db_id :database.id]- [:metabase_field :pk-field] [:and [:= :fk-field.fk_target_field_id :pk-field.id]- :pk-field.active]]- :where [:and- [:= :database.id database-id]- [:not= :fk-field.fk_target_field_id nil]- :fk-field.active]- :order-by [[:fk-field.id :desc]- [:pk-field.id :desc]]})+ (let [rows (parameters.db/fk-relationships-for-database database-id)joins (for [{:keys [t1 f1 t2 f2]} rows]{:lhs {:table t1, :field f1}:rhs {:table t2, :field f2}})diff --git a/src/metabase/parameters/db.clj b/src/metabase/parameters/db.cljindex 9f84f968a92..ddffb7538f6 100644--- a/src/metabase/parameters/db.clj+++ b/src/metabase/parameters/db.clj@@ -138,3 +138,24 @@"The `columns` of the Fields with `field-ids`."[columns field-ids](t2/select (into [:model/Field] columns) :id [:in field-ids]))++(defn fk-relationships-for-database+ "Rows describing FK -> PK Field relationships (`:f1`/`:t1` FK Field/Table ids, `:f2`/`:t2` PK Field/Table ids)+ for active Fields in the Database with `database-id`."+ [database-id]+ (mdb/query {:select [[:fk-field.id :f1]+ [:fk-table.id :t1]+ [:pk-field.id :f2]+ [:pk-field.table_id :t2]]+ :from [[:metabase_field :fk-field]]+ :left-join [[:metabase_table :fk-table] [:and [:= :fk-field.table_id :fk-table.id]+ :fk-table.active]+ [:metabase_database :database] [:= :fk-table.db_id :database.id]+ [:metabase_field :pk-field] [:and [:= :fk-field.fk_target_field_id :pk-field.id]+ :pk-field.active]]+ :where [:and+ [:= :database.id database-id]+ [:not= :fk-field.fk_target_field_id nil]+ :fk-field.active]+ :order-by [[:fk-field.id :desc]+ [:pk-field.id :desc]]}))diff --git a/src/metabase/permissions/db.clj b/src/metabase/permissions/db.cljindex 793580e141f..f91a558ad0c 100644--- a/src/metabase/permissions/db.clj+++ b/src/metabase/permissions/db.clj@@ -3,6 +3,7 @@additional logic, so the rest of the module only touches `toucan2.core` for model definitions, hydration methods,and transactions."(:require+ [metabase.app-db.core :as mdb][metabase.permissions.schema :as permissions.schema][metabase.premium-features.core :as premium-features][metabase.settings.core :as setting]@@ -260,6 +261,15 @@[rows](t2/insert! :model/Permissions rows))+(defn permission-objects-for-user+ "The Permissions objects granted, via group membership, to the User with `user-id`."+ [user-id]+ (map :object (mdb/query {:select [:p.object]+ :from [[:permissions_group_membership :pgm]]+ :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]+ [:permissions :p] [:= :p.group_id :pg.id]]+ :where [:= :pgm.user_id user-id]})))+;;; --------------------------------------------- PermissionsGroup ---------------------------------------------(defn magic-group@@ -343,6 +353,19 @@[group-id changes](t2/update! :model/PermissionsGroup group-id changes))+(defn group-member-counts+ "A map of PermissionsGroup ID to number of active members in the group. Groups with no active members have no+ entry."+ []+ (let [results (mdb/query {:select [[:pgm.group_id :group_id] [[:count :pgm.id] :members]]+ :from [[:permissions_group_membership :pgm]]+ :left-join [[:core_user :user] [:= :pgm.user_id :user.id]]+ :where [:= :user.is_active true]+ :group-by [:pgm.group_id]})]+ (zipmap+ (map :group_id results)+ (map :members results))))+;;; ---------------------------------------- PermissionsGroupMembership ----------------------------------------(defn- insert-group-memberships-from-mapping-querydiff --git a/src/metabase/permissions/models/permissions_group.clj b/src/metabase/permissions/models/permissions_group.cljindex 11cee32a358..4334f36db0c 100644--- a/src/metabase/permissions/models/permissions_group.clj+++ b/src/metabase/permissions/models/permissions_group.clj@@ -199,15 +199,7 @@"Return a map of `PermissionsGroup` ID -> number of members in the group. (This doesn't include entries for emptygroups.)"[]- (let [results (mdb/query- {:select [[:pgm.group_id :group_id] [[:count :pgm.id] :members]]- :from [[:permissions_group_membership :pgm]]- :left-join [[:core_user :user] [:= :pgm.user_id :user.id]]- :where [:= :user.is_active true]- :group-by [:pgm.group_id]})]- (zipmap- (map :group_id results)- (map :members results))))+ (permissions.db/group-member-counts))(methodical/defmethod t2/batched-hydrate [:model/PermissionsGroup :member_count]"Efficiently add `:member_count` to PermissionGroups."diff --git a/src/metabase/permissions/user.clj b/src/metabase/permissions/user.cljindex f557c3116d1..26adf7dcd79 100644--- a/src/metabase/permissions/user.clj+++ b/src/metabase/permissions/user.clj@@ -1,6 +1,6 @@(ns metabase.permissions.user(:require- [metabase.app-db.core :as app-db]+ [metabase.permissions.db :as permissions.db][metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.path :as permissions.path][metabase.permissions.published-tables :as published-tables]@@ -34,11 +34,7 @@(map permissions.path/collection-readwrite-path ((requiring-resolve 'metabase.collections.models.collection/collections-in-namespace):transforms))));; include the other Perms entries for any Group this User is in (1 DB Call)- (map :object (app-db/query {:select [:p.object]- :from [[:permissions_group_membership :pgm]]- :join [[:permissions_group :pg] [:= :pgm.group_id :pg.id]- [:permissions :p] [:= :p.group_id :pg.id]]- :where [:= :pgm.user_id user-id]})))))))+ (permissions.db/permission-objects-for-user user-id))))))(defn query-creation-capabilities"Returns a map with `:can-create-queries` and `:can-create-native-queries` for the given user,diff --git a/src/metabase/pulse/db.clj b/src/metabase/pulse/db.cljindex 21f61fd80a9..05ce83cf346 100644--- a/src/metabase/pulse/db.clj+++ b/src/metabase/pulse/db.clj@@ -2,6 +2,7 @@"Application database queries for the pulse module. Every function here is a direct Toucan 2 call with noadditional logic, so no other namespace in the module runs a query itself (model definitions still use `toucan2.core`)."(:require+ [metabase.app-db.core :as app-db][toucan2.core :as t2]))(defn card@@ -30,6 +31,24 @@[dashboard-id card-ids](t2/select-fn->pk :card_id :model/DashboardCard :dashboard_id dashboard-id :card_id [:in card-ids]))+(defn pulse-card-pairs-for-dashboard+ "Distinct `:pulse-id`/`:card-id` pairs for the Pulses (subscriptions) attached to the Dashboard with+ `dashboard-id`."+ [dashboard-id]+ (app-db/query {:select-distinct [[:p.id :pulse-id] [:pc.card_id :card-id]]+ :from [[:pulse :p]]+ :left-join [[:pulse_card :pc] [:= :p.id :pc.pulse_id]]+ :where [:= :p.dashboard_id dashboard-id]}))++(defn dashboard-card-ids-for-dashboard+ "The distinct, non-nil Card ids shown by the DashboardCards of the Dashboard with `dashboard-id`."+ [dashboard-id]+ (map :card_id (app-db/query {:select-distinct [:dc.card_id]+ :from [[:report_dashboardcard :dc]]+ :where [:and+ [:= :dc.dashboard_id dashboard-id]+ [:not= :dc.card_id nil]]})))+(defn channel"The Channel with `channel-id`, or nil."[channel-id]diff --git a/src/metabase/pulse/events/dashboard_subscription.clj b/src/metabase/pulse/events/dashboard_subscription.cljindex 987a08b5bff..421afde59e9 100644--- a/src/metabase/pulse/events/dashboard_subscription.clj+++ b/src/metabase/pulse/events/dashboard_subscription.clj@@ -1,7 +1,6 @@(ns metabase.pulse.events.dashboard-subscription(:require[clojure.set :as set]- [metabase.app-db.core :as app-db][metabase.events.core :as events][metabase.pulse.db :as pulse.db][metabase.pulse.models.pulse :as models.pulse]@@ -17,20 +16,9 @@"Updates the pulses' names and collection IDs, and syncs the PulseCards"[_ {dashboard :object}](let [dashboard-id (u/the-id dashboard)- affected (app-db/query- {:select-distinct [[:p.id :pulse-id] [:pc.card_id :card-id]]- :from [[:pulse :p]]- :left-join [[:pulse_card :pc] [:= :p.id :pc.pulse_id]]- :where [:= :p.dashboard_id dashboard-id]})]+ affected (pulse.db/pulse-card-pairs-for-dashboard dashboard-id)](when-let [pulse-ids (seq (distinct (map :pulse-id affected)))]- (let [correct-card-ids (->> (app-db/query- {:select-distinct [:dc.card_id]- :from [[:report_dashboardcard :dc]]- :where [:and- [:= :dc.dashboard_id dashboard-id]- [:not= :dc.card_id nil]]})- (map :card_id)- set)+ (let [correct-card-ids (set (pulse.db/dashboard-card-ids-for-dashboard dashboard-id))stale-card-ids (->> affected(keep :card-id)set)diff --git a/src/metabase/query_processor/db.clj b/src/metabase/query_processor/db.cljindex 8f869b80120..d3d83f47ee6 100644--- a/src/metabase/query_processor/db.clj+++ b/src/metabase/query_processor/db.clj@@ -3,6 +3,7 @@additional logic, so the rest of the module never talks to `toucan2.core` itself."(:require[java-time.api :as t]+ [metabase.app-db.core :as app-db]^{:clj-kondo/ignore [:discouraged-namespace]}[toucan2.core :as t2]))@@ -71,3 +72,12 @@"The `:id`, `:database_id`, and `:card_schema` of the Cards with `card-ids`."[card-ids](t2/select [:model/Card :id :database_id :card_schema] :id [:in card-ids]))++(defn upsert-cache-entry!+ "Insert or update the QueryCache entry for `query-hash`, setting `:results` to `results` and `:updated_at` to+ `timestamp`, and clearing `:refresh_started_at`."+ [query-hash timestamp results]+ (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}+ (constantly {:updated_at timestamp+ :results results+ :refresh_started_at nil})))diff --git a/src/metabase/query_processor/middleware/cache_backend/db.clj b/src/metabase/query_processor/middleware/cache_backend/db.cljindex 53b3a917dda..8ec65cf916b 100644--- a/src/metabase/query_processor/middleware/cache_backend/db.clj+++ b/src/metabase/query_processor/middleware/cache_backend/db.clj@@ -1,7 +1,6 @@(ns metabase.query-processor.middleware.cache-backend.db(:require[java-time.api :as t]- [metabase.app-db.core :as app-db][metabase.premium-features.core :refer [defenterprise]][metabase.query-processor.db :as query-processor.db][metabase.query-processor.middleware.cache-backend.interface :as i]@@ -119,10 +118,7 @@(let [final-results (encryption/maybe-encrypt-for-stream results)timestamp (t/offset-date-time)](try- (app-db/update-or-insert! :model/QueryCache {:query_hash query-hash}- (constantly {:updated_at timestamp- :results final-results- :refresh_started_at nil}))+ (query-processor.db/upsert-cache-entry! query-hash timestamp final-results)(catch Throwable e(log/errorf "Error saving query results to cache: %s" (ex-message e))))nil))diff --git a/src/metabase/search/db.clj b/src/metabase/search/db.cljindex c6bbc16f97e..baeec9605c4 100644--- a/src/metabase/search/db.clj+++ b/src/metabase/search/db.clj@@ -3,6 +3,7 @@additional logic, so no other namespace in the module runs a query itself (connection and transaction handling still use `toucan2.core`)."(:require[honey.sql.helpers :as sql.helpers]+ [metabase.app-db.core :as mdb][metabase.search.appdb.specialization.api :as specialization][toucan2.core :as t2]))@@ -11,6 +12,24 @@[query](t2/query query))+(defn spec-index-reducible-rows+ "A reducible of the rows matching the Honey SQL `query` built from a search model's spec by+ `metabase.search.ingestion`."+ [query]+ (mdb/streaming-reducible-query query))++(defn in-place-model-set-rows+ "The rows matching the Honey SQL `query` built by `metabase.search.in-place.legacy` to find the distinct set of+ models with results."+ [query]+ (mdb/query query))++(defn in-place-search-reducible+ "A reducible of the rows matching the full in-place search Honey SQL `query` built by+ `metabase.search.in-place.legacy`."+ [query]+ (mdb/streaming-reducible-query query))+(defn scored-search-rows"The rows matching the scored, filtered search Honey SQL `query` built by `metabase.search.appdb.core`."[query]diff --git a/src/metabase/search/in_place/legacy.clj b/src/metabase/search/in_place/legacy.cljindex d967e19509a..80b3124d7a1 100644--- a/src/metabase/search/in_place/legacy.clj+++ b/src/metabase/search/in_place/legacy.clj@@ -10,6 +10,7 @@[metabase.search.config:as search.config:refer [SearchContext SearchableModel]]+ [metabase.search.db :as search.db][metabase.search.engine :as search.engine][metabase.search.filter :as search.filter][metabase.search.in-place.filter :as search.in-place.filter]@@ -702,7 +703,7 @@(cond-> {:select [:*]:from [[^:allow-subquery {:union-all nested-queries} :dummy_alias]]}(seq ctes) (assoc :with ctes)))]- (into #{} (map :model) (some-> query mdb/query))))+ (into #{} (map :model) (some-> query search.db/in-place-model-set-rows))))(mu/defn full-search-query"Postgres 9 is not happy with the type munging it needs to do to make the union-all degenerate down to a trivial case@@ -735,7 +736,7 @@(defn- results[search-ctx](let [search-query (full-search-query search-ctx)]- (mdb/streaming-reducible-query search-query)))+ (search.db/in-place-search-reducible search-query)))(defmethod search.engine/results:search.engine/in-placediff --git a/src/metabase/search/ingestion.clj b/src/metabase/search/ingestion.cljindex 8d5c5d94c95..fc47d77df90 100644--- a/src/metabase/search/ingestion.clj+++ b/src/metabase/search/ingestion.clj@@ -203,7 +203,7 @@;; joined side has its own integrity violations. Downstream upserts hit a unique constraint on;; (model, model_id), so dedup here at the streaming boundary — bounded by per-model row count.(->> (spec-index-query-where search-model where-clause)- mdb/streaming-reducible-query+ search.db/spec-index-reducible-rows(eduction (comp (map #(assoc % :model search-model))(m/distinct-by (juxt :id :model))))))diff --git a/src/metabase/settings/db.clj b/src/metabase/settings/db.cljindex 1aab48c9714..e16ff494605 100644--- a/src/metabase/settings/db.clj+++ b/src/metabase/settings/db.clj@@ -2,6 +2,7 @@"Application database queries for the settings module. Every function here is a direct Toucan 2 call with noadditional logic, so the rest of the module only touches `toucan2.core` for model definitions."(:require+ [metabase.app-db.core :as mdb][toucan2.core :as t2]))(defn setting-value@@ -41,6 +42,11 @@(t2/insert! (t2/table-name (t2/resolve-model :model/Setting)):key setting-key, :value value, :value_with_aad value-with-aad))+(defn current-timestamp-string+ "The application DB's own current timestamp, as a string."+ []+ (mdb/current-timestamp-string (mdb/db-type)))+(defn update-user-settings!"Store `settings-json` as the user-local settings of the User with `user-id`."[user-id settings-json]diff --git a/src/metabase/settings/models/setting/cache.clj b/src/metabase/settings/models/setting/cache.cljindex faf5913e670..a98badf5d1b 100644--- a/src/metabase/settings/models/setting/cache.clj+++ b/src/metabase/settings/models/setting/cache.clj@@ -75,7 +75,7 @@(log/debug "Updating value of settings-last-updated in DB...");; Written raw, not through `:model/Setting`, so that `value` gets the plaintext timestamp a version predating;; `value_with_aad` compares in SQL. `value_with_aad` is encrypted under the marker's AAD like any other setting's.- (let [value (mdb/current-timestamp-string (mdb/db-type))+ (let [value (settings.db/current-timestamp-string)value-with-aad (encryption/maybe-encrypt value {:aad (mdb.setting/setting-aad settings-last-updated-key)})];; attempt to UPDATE the existing row. If no row exists, `t2/update!` will return 0...(or (pos? (settings.db/update-raw-setting-row! settings-last-updated-key value value-with-aad))diff --git a/src/metabase/sync/analyze/data_sensitivity.clj b/src/metabase/sync/analyze/data_sensitivity.cljindex 55503fdfa3d..da8b7de2e38 100644--- a/src/metabase/sync/analyze/data_sensitivity.clj+++ b/src/metabase/sync/analyze/data_sensitivity.clj@@ -8,6 +8,7 @@change can reach fields that already carry a category."(:require[metabase.analyze.core :as analyze]+ [metabase.sync.db :as sync.db][metabase.sync.interface :as i][metabase.sync.settings :as sync.settings][metabase.sync.util :as sync-util]@@ -34,11 +35,6 @@[:force? {:optional true} [:maybe :boolean]][:ignore-setting? {:optional true} [:maybe :boolean]]])-(defn- selection-clause [force?]- (if force?- [:or [:= :data_sensitivity nil] [:= :data_sensitivity "PUBLIC"]]- [:= :data_sensitivity nil]))-(defn- skip? [ignore-setting?](and (not ignore-setting?)(not (sync.settings/data-sensitivity-scan-enabled))))@@ -46,13 +42,7 @@(mu/defn- fields-to-scan :- [:sequential i/FieldInstance][table :- i/TableInstanceforce?]- (t2/select :model/Field- {:where [:and- [:= :table_id (u/the-id table)]- [:= :active true]- [:not= :visibility_type "retired"]- (selection-clause force?)]- :order-by [[:id :asc]]}))+ (sync.db/fields-to-scan-for-data-sensitivity (u/the-id table) force?))(mu/defn- classify-and-save!"Returns the inferred category, nil when no rule matched (in which case `:PUBLIC` was written), or the Exception."@@ -60,7 +50,7 @@table :- i/TableInstance](sync-util/with-error-handling (format "Error classifying data sensitivity for %s" (sync-util/name-for-logging field))(let [category (analyze/infer-data-sensitivity field {:name (:name table) :entity_type (:entity_type table)})]- (t2/update! :model/Field (u/the-id field) {:data_sensitivity (or category :PUBLIC)})+ (sync.db/update-field-data-sensitivity! (u/the-id field) (or category :PUBLIC))category)))(mu/defn scan-table! :- Stats@@ -88,17 +78,7 @@(mu/defn- table-ids-with-unscanned-fields :- [:maybe [:set pos-int?]][database :- i/DatabaseInstanceforce?]- (t2/select-fn-set :table_id :model/Field- {:select [[:metabase_field.table_id :table_id]]- :from [:metabase_field]- :join [[:metabase_table] [:= :metabase_field.table_id :metabase_table.id]]- :where [:and- [:= :metabase_table.db_id (u/the-id database)]- [:= :metabase_table.active true]- [:= :metabase_field.active true]- [:not= :metabase_field.visibility_type "retired"]- (selection-clause force?)]- :group-by [:metabase_field.table_id]}))+ (sync.db/table-ids-with-fields-to-scan-for-data-sensitivity (u/the-id database) force?))(mu/defn scan-fields-for-db! :- Stats"Label every unscanned Field in every active table of `database`. `log-fn` is accepted for parity with the other@@ -115,35 +95,16 @@(map #(scan-table! % :force? force? :ignore-setting? true)))(partial merge-with +)zero-stats- (t2/reducible-select :model/Table- :id [:in table-ids]- {:order-by [[:schema :asc] [:name :asc]]}))))))--(defn- scope-clause [database-or-table]- (case (t2/model database-or-table)- :model/Table [:= :table_id (u/the-id database-or-table)]- :model/Database [:in :table_id ^:allow-subquery {:select [:id]- :from [:metabase_table]- :where [:= :db_id (u/the-id database-or-table)]}]))--(def ^:private classifier-label-clause- "A non-null `data_sensitivity` with no value in the `FieldUserSettings` mirror was written by the classifier."- [:and- [:not= :data_sensitivity nil]- [:not [:exists ^:allow-subquery {:select [1]- :from [[:metabase_field_user_settings :s]]- :where [:and- [:= :s.field_id :metabase_field.id]- [:not= :s.data_sensitivity nil]]}]]])+ (sync.db/tables-by-schema-and-name-reducible table-ids))))))(mu/defn reset-data-sensitivity! :- :int"REPL entry point: clear every classifier-written `data_sensitivity` in a Database or a single Table so the nextscan recomputes them, including fields that carry a category and would otherwise never be reselected. Labels witha value in the `FieldUserSettings` mirror are human-set and untouched. Returns the number of fields cleared."[database-or-table :- [:or i/DatabaseInstance i/TableInstance]]- (let [n (t2/query-one {:update :metabase_field- :set {:data_sensitivity nil}- :where [:and (scope-clause database-or-table) classifier-label-clause]})]+ (let [n (case (t2/model database-or-table)+ :model/Table (sync.db/reset-classifier-data-sensitivity-for-table! (u/the-id database-or-table))+ :model/Database (sync.db/reset-classifier-data-sensitivity-for-database! (u/the-id database-or-table)))](log/infof "Data sensitivity reset %d classifier-written labels in %s" n (sync-util/name-for-logging database-or-table))n))diff --git a/src/metabase/sync/db.clj b/src/metabase/sync/db.cljindex 08533d0159c..7d80eebe165 100644--- a/src/metabase/sync/db.clj+++ b/src/metabase/sync/db.clj@@ -513,6 +513,85 @@(t2/query-one (mark-fk-statement db-id fk-table-schema fk-table-name fk-column-namepk-table-schema pk-table-name pk-column-name)))+;;; ------------------------------------------ Field data sensitivity ------------------------------------------++(defn- data-sensitivity-to-scan-clause+ "Honey SQL clause matching Fields the data-sensitivity classifier still has to scan: unlabeled ones, plus those it+ labeled `PUBLIC` when `rescan-public?`."+ [rescan-public?]+ (if rescan-public?+ [:or [:= :data_sensitivity nil] [:= :data_sensitivity "PUBLIC"]]+ [:= :data_sensitivity nil]))++(defn fields-to-scan-for-data-sensitivity+ "The active, non-retired Fields of the Table with `table-id` that the data-sensitivity classifier still has to scan+ (see [[data-sensitivity-to-scan-clause]]), ordered by ID."+ [table-id rescan-public?]+ (t2/select :model/Field+ {:where [:and+ [:= :table_id table-id]+ [:= :active true]+ [:not= :visibility_type "retired"]+ (data-sensitivity-to-scan-clause rescan-public?)]+ :order-by [[:id :asc]]}))++(defn table-ids-with-fields-to-scan-for-data-sensitivity+ "The IDs of the active Tables of the Database with `database-id` that have active, non-retired Fields the+ data-sensitivity classifier still has to scan (see [[data-sensitivity-to-scan-clause]])."+ [database-id rescan-public?]+ (t2/select-fn-set :table_id :model/Field+ {:select [[:metabase_field.table_id :table_id]]+ :from [:metabase_field]+ :join [[:metabase_table] [:= :metabase_field.table_id :metabase_table.id]]+ :where [:and+ [:= :metabase_table.db_id database-id]+ [:= :metabase_table.active true]+ [:= :metabase_field.active true]+ [:not= :metabase_field.visibility_type "retired"]+ (data-sensitivity-to-scan-clause rescan-public?)]+ :group-by [:metabase_field.table_id]}))++(defn tables-by-schema-and-name-reducible+ "Reducible Tables with `table-ids`, ordered by schema and name."+ [table-ids]+ (t2/reducible-select :model/Table :id [:in table-ids] {:order-by [[:schema :asc] [:name :asc]]}))++(defn update-field-data-sensitivity!+ "Set the `data_sensitivity` of the Field with `field-id` to `data-sensitivity`."+ [field-id data-sensitivity]+ (t2/update! :model/Field field-id {:data_sensitivity data-sensitivity}))++(def ^:private classifier-data-sensitivity-clause+ "Honey SQL clause matching Fields whose non-null `data_sensitivity` has no value in the `FieldUserSettings` mirror,+ i.e. was written by the data-sensitivity classifier rather than a user."+ [:and+ [:not= :data_sensitivity nil]+ [:not [:exists ^:allow-subquery {:select [1]+ :from [[:metabase_field_user_settings :s]]+ :where [:and+ [:= :s.field_id :metabase_field.id]+ [:not= :s.data_sensitivity nil]]}]]])++(defn reset-classifier-data-sensitivity-for-table!+ "Clear the classifier-written `data_sensitivity` (see [[classifier-data-sensitivity-clause]]) of the Fields of the+ Table with `table-id`. Returns the number of Fields cleared."+ [table-id]+ (t2/query-one {:update :metabase_field+ :set {:data_sensitivity nil}+ :where [:and [:= :table_id table-id] classifier-data-sensitivity-clause]}))++(defn reset-classifier-data-sensitivity-for-database!+ "Clear the classifier-written `data_sensitivity` (see [[classifier-data-sensitivity-clause]]) of the Fields of every+ Table of the Database with `database-id`. Returns the number of Fields cleared."+ [database-id]+ (t2/query-one {:update :metabase_field+ :set {:data_sensitivity nil}+ :where [:and+ [:in :table_id ^:allow-subquery {:select [:id]+ :from [:metabase_table]+ :where [:= :db_id database-id]}]+ classifier-data-sensitivity-clause]}))+;;; ---------------------------------------------- FieldValues ----------------------------------------------(defn field-values-exist?diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.cljindex 9d9f2ae2968..03d8a7c382e 100644--- a/src/metabase/transforms/models/transform_run.clj+++ b/src/metabase/transforms/models/transform_run.clj@@ -38,7 +38,7 @@(boolean (when-let [transform-id (:transform_id instance)](mi/can-read? :model/Transform transform-id)))))([_model pk]- (when-let [run (t2/select-one :model/TransformRun :id pk)]+ (when-let [run (transforms.db/run pk)](mi/can-read? run))))(mi/define-simple-hydration-method add-transform-runsdiff --git a/src/metabase/warehouse_schema/db.clj b/src/metabase/warehouse_schema/db.cljindex 8de4de6e3ab..5eff57a4c7a 100644--- a/src/metabase/warehouse_schema/db.clj+++ b/src/metabase/warehouse_schema/db.clj@@ -225,6 +225,15 @@[field-values-id changes](t2/update! :model/FieldValues field-values-id changes))+(defn find-or-insert-full-field-values!+ "The full FieldValues of the Field with `field-id`, inserting one with `has-more-values`, `values`, and no+ `human_readable_values` if none exists yet."+ [field-id has-more-values values]+ (app-db/select-or-insert! :model/FieldValues {:field_id field-id, :type :full}+ (constantly {:has_more_values has-more-values+ :values values+ :human_readable_values nil})))+(defn touch-field-values!"Stamp `last_used_at` on the FieldValues with `field-values-id`."[field-values-id]diff --git a/src/metabase/warehouse_schema/models/field_values.clj b/src/metabase/warehouse_schema/models/field_values.cljindex 06094e891a4..81ab782c284 100644--- a/src/metabase/warehouse_schema/models/field_values.clj+++ b/src/metabase/warehouse_schema/models/field_values.clj@@ -27,7 +27,6 @@[java-time.api :as t][medley.core :as m][metabase.analyze.core :as analyze]- [metabase.app-db.core :as app-db][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.schema.metadata :as lib.schema.metadata]@@ -478,10 +477,7 @@(nil? existing-fv)(do(log/debugf "Storing FieldValues for Field %s..." field-name)- (app-db/select-or-insert! :model/FieldValues {:field_id (u/the-id field), :type :full}- (constantly {:has_more_values has-more-values- :values values- :human_readable_values nil}))+ (warehouse-schema.db/find-or-insert-full-field-values! (u/the-id field) has-more-values values)::fv-created);; if existing FieldValues won't change, skip itdiff --git a/src/metabase/warehouse_schema_rest/api/field.clj b/src/metabase/warehouse_schema_rest/api/field.cljindex 8868c9e17f2..bc094549bc4 100644--- a/src/metabase/warehouse_schema_rest/api/field.clj+++ b/src/metabase/warehouse_schema_rest/api/field.clj@@ -3,7 +3,6 @@[metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.api.macros :as api.macros]- [metabase.app-db.core :as app-db][metabase.events.core :as events][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.models.interface :as mi]@@ -323,11 +322,11 @@[400 (str "You can only update the human readable values of a mapped values of a Field whose value of ""`has_field_values` is `list` or whose 'base_type' is 'type/Boolean'.")])(let [human-readable-values? (validate-human-readable-pairs value-pairs)- update-map {:values (map first value-pairs)- :human_readable_values (when human-readable-values?- (map second value-pairs))}- updated-pk (app-db/update-or-insert! :model/FieldValues {:field_id (u/the-id field), :type :full}- (constantly update-map))]+ values (map first value-pairs)+ human-readable-values (when human-readable-values?+ (map second value-pairs))+ updated-pk (warehouse-schema-rest.db/update-or-insert-full-field-values!+ (u/the-id field) values human-readable-values)](api/check-500 (pos? updated-pk)))){:status :success})diff --git a/src/metabase/warehouse_schema_rest/db.clj b/src/metabase/warehouse_schema_rest/db.cljindex 1f6e6a554b0..5fea0e2944c 100644--- a/src/metabase/warehouse_schema_rest/db.clj+++ b/src/metabase/warehouse_schema_rest/db.clj@@ -157,3 +157,11 @@"Delete the FieldValues of the Fields with `field-ids`."[field-ids](t2/delete! (t2/table-name :model/FieldValues) :field_id [:in field-ids]))++(defn update-or-insert-full-field-values!+ "Update the full FieldValues of the Field with `field-id` to have `values` and `human-readable-values`, inserting+ one if none exists yet. Returns the number of rows affected."+ [field-id values human-readable-values]+ (app-db/update-or-insert! :model/FieldValues {:field_id field-id, :type :full}+ (constantly {:values values+ :human_readable_values human-readable-values})))diff --git a/src/metabase/warehouses_rest/api.clj b/src/metabase/warehouses_rest/api.cljindex 20b1fc2d65f..55af4910680 100644--- a/src/metabase/warehouses_rest/api.clj+++ b/src/metabase/warehouses_rest/api.clj@@ -6,7 +6,6 @@[metabase.analytics.core :as analytics][metabase.api.common :as api][metabase.api.macros :as api.macros]- [metabase.app-db.core :as mdb][metabase.classloader.core :as classloader][metabase.config.core :as config][metabase.database-routing.core :as database-routing]@@ -504,52 +503,6 @@:include-editable-data-model? include_editable_data_model:exclude-uneditable-details? exclude_uneditable_details}))-(def ^:private database-usage-models- "List of models that are used to report usage on a database."- [:question :dataset :metric :segment :transform]) ; TODO -- rename `:dataset` to `:model`?--(defmulti ^:private database-usage-query- "Query that will returns the number of `model` that use the database with id `database-id`.- The query must returns a scalar, and the method could return `nil` in case no query is available."- {:arglists '([model database-id])}- (fn [model _database-id] (keyword model)))--(defn- card-query- [db-id model type-str]- ^:allow-subquery {:select [[:%count.* model]]- :from [:report_card]- :where [:and- [:= :database_id db-id]- [:= :type type-str]]})--(defmethod database-usage-query :question- [_ db-id]- (card-query db-id :question "question"))--(defmethod database-usage-query :dataset- [_ db-id]- (card-query db-id :dataset "model"))--(defmethod database-usage-query :metric- [_ db-id]- (card-query db-id :metric "metric"))--(defmethod database-usage-query :segment- [_ db-id]- ^:allow-subquery {:select [[:%count.* :segment]]- :from [:segment]- :where [:in :table_id ^:allow-subquery {:select [:id]- :from [:metabase_table]- :where [:= :db_id db-id]}]})--(defmethod database-usage-query :transform- [_ db-id]- ^:allow-subquery {:select [[:%count.* :transform]]- :from [:transform]- :where [:or- [:= :source_database_id db-id]- [:= :target_db_id db-id]]})-;; TODO (Cam 10/28/25) -- fix this endpoint route to use kebab-case for consistency with the rest of our REST API;;;; TODO (Cam 2025-11-25) please add a response schema to this API endpoint, it makes it easier for our customers to@@ -564,12 +517,7 @@[:id ms/PositiveInt]]](api/check-superuser)(check-database-exists id)- (first (mdb/query- {:select [:*]- :from (for [model database-usage-models- :let [query (database-usage-query model id)]- :when query]- [query model])})))+ (first (warehouses-rest.db/database-usage-counts id)));;; ----------------------------------------- GET /api/database/:id/metadata -----------------------------------------diff --git a/src/metabase/warehouses_rest/db.clj b/src/metabase/warehouses_rest/db.cljindex d8f2fefbf23..b741540f44f 100644--- a/src/metabase/warehouses_rest/db.clj+++ b/src/metabase/warehouses_rest/db.clj@@ -287,3 +287,33 @@"The ids of the Collections named `collection-name`, or nil."[collection-name](t2/select-pks-set :model/Collection :name collection-name))++(defn- card-usage-count-subquery+ [database-id model type-str]+ ^:allow-subquery {:select [[:%count.* model]]+ :from [:report_card]+ :where [:and+ [:= :database_id database-id]+ [:= :type type-str]]})++(defn database-usage-counts+ "A single row with the count of Questions (`:question`), Models (`:dataset`), Metrics (`:metric`), Segments+ (`:segment`), and Transforms (`:transform`) that use the Database with `database-id`."+ [database-id]+ (mdb/query+ {:select [:*]+ :from [[(card-usage-count-subquery database-id :question "question") :question]+ [(card-usage-count-subquery database-id :dataset "model") :dataset]+ [(card-usage-count-subquery database-id :metric "metric") :metric]+ [^:allow-subquery {:select [[:%count.* :segment]]+ :from [:segment]+ :where [:in :table_id ^:allow-subquery {:select [:id]+ :from [:metabase_table]+ :where [:= :db_id database-id]}]}+ :segment]+ [^:allow-subquery {:select [[:%count.* :transform]]+ :from [:transform]+ :where [:or+ [:= :source_database_id database-id]+ [:= :target_db_id database-id]]}+ :transform]]}))